diff --git a/.devcontainer/Dockerfile.trtllm b/.devcontainer/Dockerfile.trtllm
new file mode 100644
index 00000000..e69de29b
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
new file mode 100644
index 00000000..e69de29b
diff --git a/.dockerignore b/.dockerignore
index c69283ec..0b495924 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -2,3 +2,6 @@ aml
 target
 server/transformers
 server/flash-attention
+cmake-build-debug/
+cmake-build-release/
+Dockerfile*
diff --git a/.gitignore b/.gitignore
index e9ad1808..4270a1ae 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,9 +3,12 @@ target
 router/tokenizer.json
 *__pycache__*
 
+backends/v2/src/client/pb
+backends/v3/src/client/pb
+
 # ROCm auto-generated files
 *.hip
-server/exllamav2_kernels/exllamav2_kernels/hip/
+server/exllamav2
 server/exllama_kernels/exllama_kernels/hip/
 server/exllama_kernels/exllama_kernels/hip_func/
 *_hip.cuh
@@ -14,3 +17,7 @@ server/exllama_kernels/exllama_kernels/exllama_ext_hip.cpp
 
 data/
 load_tests/*.json
+server/fbgemmm
+
+.direnv/
+.venv/
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 45bc07a5..0c8b6885 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -5,7 +5,7 @@ repos:
     -   id: check-yaml
     -   id: end-of-file-fixer
     -   id: trailing-whitespace
-        exclude: docs/source/basic_tutorials/launcher.md
+        exclude: docs/source/reference/launcher.md
 -   repo: https://github.com/psf/black
     rev: 24.2.0
     hooks:
@@ -13,6 +13,11 @@ repos:
 -   repo: https://github.com/doublify/pre-commit-rust
     rev: v1.0
     hooks:
-    -   id: fmt
     -   id: cargo-check
+    -   id: fmt
     -   id: clippy
+-   repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.3.0
+    hooks:
+    -   id: ruff
+        args: [--fix, --exit-non-zero-on-fix]
diff --git a/.redocly.lint-ignore.yaml b/.redocly.lint-ignore.yaml
new file mode 100644
index 00000000..fb02c00f
--- /dev/null
+++ b/.redocly.lint-ignore.yaml
@@ -0,0 +1,82 @@
+# This file instructs Redocly's linter to ignore the rules contained for specific parts of your API.
+# See https://redoc.ly/docs/cli/ for more information.
+docs/openapi.json:
+  no-empty-servers:
+    - '#/openapi'
+  spec:
+    - >-
+      #/components/schemas/GenerateParameters/properties/best_of/exclusiveMinimum
+    - >-
+      #/components/schemas/GenerateParameters/properties/frequency_penalty/exclusiveMinimum
+    - '#/components/schemas/GenerateParameters/properties/grammar/nullable'
+    - >-
+      #/components/schemas/GenerateParameters/properties/repetition_penalty/exclusiveMinimum
+    - '#/components/schemas/GenerateParameters/properties/seed/exclusiveMinimum'
+    - >-
+      #/components/schemas/GenerateParameters/properties/temperature/exclusiveMinimum
+    - '#/components/schemas/GenerateParameters/properties/top_k/exclusiveMinimum'
+    - >-
+      #/components/schemas/GenerateParameters/properties/top_n_tokens/exclusiveMinimum
+    - '#/components/schemas/GenerateParameters/properties/top_p/exclusiveMinimum'
+    - >-
+      #/components/schemas/GenerateParameters/properties/typical_p/exclusiveMinimum
+    - '#/components/schemas/GenerateResponse/properties/details/nullable'
+    - '#/components/schemas/StreamResponse/properties/details/nullable'
+    - '#/components/schemas/ChatRequest/properties/response_format/nullable'
+    - '#/components/schemas/ChatRequest/properties/stream_options/nullable'
+    - '#/components/schemas/ChatRequest/properties/tool_choice/nullable'
+    - '#/components/schemas/ToolChoice/nullable'
+    - '#/components/schemas/ChatCompletionComplete/properties/logprobs/nullable'
+    - '#/components/schemas/ChatCompletionChunk/properties/usage/nullable'
+    - '#/components/schemas/ChatCompletionChoice/properties/logprobs/nullable'
+  no-invalid-media-type-examples:
+    - '#/paths/~1/post/responses/422/content/application~1json/example'
+    - '#/paths/~1/post/responses/424/content/application~1json/example'
+    - '#/paths/~1/post/responses/429/content/application~1json/example'
+    - '#/paths/~1/post/responses/500/content/application~1json/example'
+    - '#/paths/~1generate/post/responses/422/content/application~1json/example'
+    - '#/paths/~1generate/post/responses/424/content/application~1json/example'
+    - '#/paths/~1generate/post/responses/429/content/application~1json/example'
+    - '#/paths/~1generate/post/responses/500/content/application~1json/example'
+    - >-
+      #/paths/~1generate_stream/post/responses/422/content/text~1event-stream/example
+    - >-
+      #/paths/~1generate_stream/post/responses/424/content/text~1event-stream/example
+    - >-
+      #/paths/~1generate_stream/post/responses/429/content/text~1event-stream/example
+    - >-
+      #/paths/~1generate_stream/post/responses/500/content/text~1event-stream/example
+    - '#/paths/~1tokenize/post/responses/404/content/application~1json/example'
+    - >-
+      #/paths/~1v1~1chat~1completions/post/responses/422/content/application~1json/example
+    - >-
+      #/paths/~1v1~1chat~1completions/post/responses/424/content/application~1json/example
+    - >-
+      #/paths/~1v1~1chat~1completions/post/responses/429/content/application~1json/example
+    - >-
+      #/paths/~1v1~1chat~1completions/post/responses/500/content/application~1json/example
+    - >-
+      #/paths/~1v1~1completions/post/responses/422/content/application~1json/example
+    - >-
+      #/paths/~1v1~1completions/post/responses/424/content/application~1json/example
+    - >-
+      #/paths/~1v1~1completions/post/responses/429/content/application~1json/example
+    - >-
+      #/paths/~1v1~1completions/post/responses/500/content/application~1json/example
+  operation-4xx-response:
+    - '#/paths/~1health/get/responses'
+    - '#/paths/~1info/get/responses'
+    - '#/paths/~1metrics/get/responses'
+  no-unused-components:
+    - '#/components/schemas/Completion'
+  security-defined:
+    - '#/paths/~1/post'
+    - '#/paths/~1generate/post'
+    - '#/paths/~1generate_stream/post'
+    - '#/paths/~1health/get'
+    - '#/paths/~1info/get'
+    - '#/paths/~1metrics/get'
+    - '#/paths/~1tokenize/post'
+    - '#/paths/~1v1~1chat~1completions/post'
+    - '#/paths/~1v1~1completions/post'
+    - '#/paths/~1v1~1models/get'
diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md
new file mode 100644
index 00000000..b23f3150
--- /dev/null
+++ b/CODE_OF_CONDUCT.md
@@ -0,0 +1,133 @@
+
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, caste, color, religion, or sexual
+identity and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our
+community include:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes,
+  and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the overall
+  community
+
+Examples of unacceptable behavior include:
+
+* The use of sexualized language or imagery, and sexual attention or advances of
+  any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or email address,
+  without their explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official e-mail address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the community leaders responsible for enforcement at
+feedback@huggingface.co.
+All complaints will be reviewed and investigated promptly and fairly.
+
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series of
+actions.
+
+**Consequence**: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or permanent
+ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including
+sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior, harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within the
+community.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.1, available at
+[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1].
+
+Community Impact Guidelines were inspired by
+[Mozilla's code of conduct enforcement ladder][Mozilla CoC].
+
+For answers to common questions about this code of conduct, see the FAQ at
+[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at
+[https://www.contributor-covenant.org/translations][translations].
+
+[homepage]: https://www.contributor-covenant.org
+[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html
+[Mozilla CoC]: https://github.com/mozilla/diversity
+[FAQ]: https://www.contributor-covenant.org/faq
+[translations]: https://www.contributor-covenant.org/translations
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
new file mode 100644
index 00000000..d541e47f
--- /dev/null
+++ b/CONTRIBUTING.md
@@ -0,0 +1,120 @@
+<!---
+Copyright 2024 The HuggingFace Team. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+# Contribute to text-generation-inference
+
+Everyone is welcome to contribute, and we value everybody's contribution. Code
+contributions are not the only way to help the community. Answering questions, helping
+others, and improving the documentation are also immensely valuable.
+
+It also helps us if you spread the word! Reference the library in blog posts
+about the awesome projects it made possible, shout out on Twitter every time it has
+helped you, or simply ⭐️ the repository to say thank you.
+
+However you choose to contribute, please be mindful and respect our
+[code of conduct](https://github.com/huggingface/text-generation-inference/blob/main/CODE_OF_CONDUCT.md).
+
+**This guide was heavily inspired by the awesome [scikit-learn guide to contributing](https://github.com/scikit-learn/scikit-learn/blob/main/CONTRIBUTING.md).**
+
+## Ways to contribute
+
+There are several ways you can contribute to text-generation-inference.
+
+* Fix outstanding issues with the existing code.
+* Submit issues related to bugs or desired new features.
+* Contribute to the examples or to the documentation.
+
+> All contributions are equally valuable to the community. 🥰
+
+## Fixing outstanding issues
+
+If you notice an issue with the existing code and have a fix in mind, feel free to [start contributing](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request) and open
+a Pull Request!
+
+## Submitting a bug-related issue or feature request
+
+Do your best to follow these guidelines when submitting a bug-related issue or a feature
+request. It will make it easier for us to come back to you quickly and with good
+feedback.
+
+### Did you find a bug?
+
+The text-generation-inference library is robust and reliable thanks to users who report the problems they encounter.
+
+Before you report an issue, we would really appreciate it if you could **make sure the bug was not
+already reported** (use the search bar on GitHub under Issues). Your issue should also be related to bugs in the
+library itself, and not your code.
+
+Once you've confirmed the bug hasn't already been reported, please include the following information in your issue so
+we can quickly resolve it:
+
+* Your **OS type and version**, as well as your environment versions (versions of rust, python, and dependencies).
+* A short, self-contained, code snippet that allows us to reproduce the bug.
+* The *full* traceback if an exception is raised.
+* Attach any other additional information, like screenshots, you think may help.
+
+To get the OS and software versions automatically, you can re-run the launcher with the `--env` flag:
+
+```bash
+text-generation-launcher --env
+```
+
+This will precede the launch of the model with the information relative to your environment. We recommend pasting
+that in your issue report.
+
+### Do you want a new feature?
+
+If there is a new feature you'd like to see in text-generation-inference, please open an issue and describe:
+
+1. What is the *motivation* behind this feature? Is it related to a problem or frustration with the library? Is it
+   a feature related to something you need for a project? Is it something you worked on and think it could benefit
+   the community?
+
+   Whatever it is, we'd love to hear about it!
+
+2. Describe your requested feature in as much detail as possible. The more you can tell us about it, the better
+   we'll be able to help you.
+3. Provide a *code snippet* that demonstrates the feature's usage.
+4. If the feature is related to a paper, please include a link.
+
+If your issue is well written we're already 80% of the way there by the time you create it.
+
+We have added [templates](https://github.com/huggingface/text-generation-inference/tree/main/.github/ISSUE_TEMPLATE)
+to help you get started with your issue.
+
+## Do you want to implement a new model?
+
+New models are constantly released and if you want to implement a new model, please provide the following information:
+
+* A short description of the model and a link to the paper.
+* Link to the implementation if it is open-sourced.
+* Link to the model weights if they are available.
+
+If you are willing to contribute the model yourself, let us know so we can help you add it to text-generation-inference!
+
+## Do you want to add documentation?
+
+We're always looking for improvements to the documentation that make it more clear and accurate. Please let us know
+how the documentation can be improved such as typos and any content that is missing, unclear or inaccurate. We'll be
+happy to make the changes or help you make a contribution if you're interested!
+
+## I want to become a maintainer of the project. How do I get there?
+
+TGI is a project led and managed by Hugging Face as it powers our internal services. However, we are happy to have
+motivated individuals from other organizations join us as maintainers with the goal of making TGI the best inference
+service.
+
+If you are such an individual (or organization), please reach out to us and let's collaborate.
diff --git a/Cargo.lock b/Cargo.lock
index 2e75fe8f..27499cd4 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4,9 +4,9 @@ version = 3
 
 [[package]]
 name = "addr2line"
-version = "0.22.0"
+version = "0.24.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678"
+checksum = "f5fb1d8e4442bd405fdfd1dacb42792696b0cf9cb15882e5d097b742a676d375"
 dependencies = [
  "gimli",
 ]
@@ -17,6 +17,12 @@ version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
 
+[[package]]
+name = "adler2"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627"
+
 [[package]]
 name = "ahash"
 version = "0.8.11"
@@ -47,10 +53,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4aa90d7ce82d4be67b64039a3d588d38dbcc6736577de4a847025ce5b0c468d1"
 
 [[package]]
-name = "anstream"
-version = "0.6.14"
+name = "allocator-api2"
+version = "0.2.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "418c75fa768af9c03be99d17643f93f79bbba589895012a80e3452a19ddda15b"
+checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f"
+
+[[package]]
+name = "anstream"
+version = "0.6.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "64e15c1ab1f89faffbf04a634d5e1962e9074f2741eef6d97f3c4e322426d526"
 dependencies = [
  "anstyle",
  "anstyle-parse",
@@ -63,33 +75,33 @@ dependencies = [
 
 [[package]]
 name = "anstyle"
-version = "1.0.7"
+version = "1.0.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "038dfcf04a5feb68e9c60b21c9625a54c2c0616e79b72b0fd87075a056ae1d1b"
+checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1"
 
 [[package]]
 name = "anstyle-parse"
-version = "0.2.4"
+version = "0.2.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c03a11a9034d92058ceb6ee011ce58af4a9bf61491aa7e1e59ecd24bd40d22d4"
+checksum = "eb47de1e80c2b463c735db5b217a0ddc39d612e7ac9e2e96a5aed1f57616c1cb"
 dependencies = [
  "utf8parse",
 ]
 
 [[package]]
 name = "anstyle-query"
-version = "1.0.3"
+version = "1.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a64c907d4e79225ac72e2a354c9ce84d50ebb4586dee56c82b3ee73004f537f5"
+checksum = "6d36fc52c7f6c869915e99412912f22093507da8d9e942ceaf66fe4b7c14422a"
 dependencies = [
  "windows-sys 0.52.0",
 ]
 
 [[package]]
 name = "anstyle-wincon"
-version = "3.0.3"
+version = "3.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "61a38449feb7068f52bb06c12759005cf459ee52bb4adc1d5a7c4322d716fb19"
+checksum = "5bf74e1b6e971609db8ca7a9ce79fd5768ab6ae46441c572e46cf596f59e57f8"
 dependencies = [
  "anstyle",
  "windows-sys 0.52.0",
@@ -97,9 +109,9 @@ dependencies = [
 
 [[package]]
 name = "anyhow"
-version = "1.0.86"
+version = "1.0.89"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da"
+checksum = "86fdf8605db99b54d3cd748a44c6d04df638eb5dafb219b135d0149bd0db01f6"
 
 [[package]]
 name = "arbitrary"
@@ -121,14 +133,14 @@ checksum = "0ae92a5119aa49cdbcf6b9f893fe4e1d98b04ccbf82ee0584ad948a44a734dea"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.79",
 ]
 
 [[package]]
 name = "arrayvec"
-version = "0.7.4"
+version = "0.7.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711"
+checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
 
 [[package]]
 name = "async-rustls"
@@ -160,25 +172,42 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.79",
 ]
 
 [[package]]
 name = "async-trait"
-version = "0.1.80"
+version = "0.1.83"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca"
+checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.79",
+]
+
+[[package]]
+name = "atomic-waker"
+version = "1.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"
+
+[[package]]
+name = "atty"
+version = "0.2.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
+dependencies = [
+ "hermit-abi 0.1.19",
+ "libc",
+ "winapi",
 ]
 
 [[package]]
 name = "autocfg"
-version = "1.3.0"
+version = "1.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0"
+checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26"
 
 [[package]]
 name = "av1-grain"
@@ -226,6 +255,33 @@ dependencies = [
  "slotmap",
 ]
 
+[[package]]
+name = "aws-lc-rs"
+version = "1.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2f95446d919226d587817a7d21379e6eb099b97b45110a7f272a444ca5c54070"
+dependencies = [
+ "aws-lc-sys",
+ "mirai-annotations",
+ "paste",
+ "zeroize",
+]
+
+[[package]]
+name = "aws-lc-sys"
+version = "0.21.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b3ddc4a5b231dd6958b140ff3151b6412b3f4321fab354f399eec8f14b06df62"
+dependencies = [
+ "bindgen",
+ "cc",
+ "cmake",
+ "dunce",
+ "fs_extra",
+ "libc",
+ "paste",
+]
+
 [[package]]
 name = "axum"
 version = "0.6.20"
@@ -233,13 +289,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf"
 dependencies = [
  "async-trait",
- "axum-core",
+ "axum-core 0.3.4",
  "bitflags 1.3.2",
  "bytes",
  "futures-util",
- "http",
- "http-body",
- "hyper",
+ "http 0.2.12",
+ "http-body 0.4.6",
+ "hyper 0.14.30",
  "itoa",
  "matchit",
  "memchr",
@@ -251,13 +307,47 @@ dependencies = [
  "serde_json",
  "serde_path_to_error",
  "serde_urlencoded",
- "sync_wrapper",
+ "sync_wrapper 0.1.2",
  "tokio",
- "tower",
+ "tower 0.4.13",
  "tower-layer",
  "tower-service",
 ]
 
+[[package]]
+name = "axum"
+version = "0.7.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "504e3947307ac8326a5437504c517c4b56716c9d98fac0028c2acc7ca47d70ae"
+dependencies = [
+ "async-trait",
+ "axum-core 0.4.5",
+ "bytes",
+ "futures-util",
+ "http 1.1.0",
+ "http-body 1.0.1",
+ "http-body-util",
+ "hyper 1.4.1",
+ "hyper-util",
+ "itoa",
+ "matchit",
+ "memchr",
+ "mime",
+ "percent-encoding",
+ "pin-project-lite",
+ "rustversion",
+ "serde",
+ "serde_json",
+ "serde_path_to_error",
+ "serde_urlencoded",
+ "sync_wrapper 1.0.1",
+ "tokio",
+ "tower 0.5.1",
+ "tower-layer",
+ "tower-service",
+ "tracing",
+]
+
 [[package]]
 name = "axum-core"
 version = "0.3.4"
@@ -267,8 +357,8 @@ dependencies = [
  "async-trait",
  "bytes",
  "futures-util",
- "http",
- "http-body",
+ "http 0.2.12",
+ "http-body 0.4.6",
  "mime",
  "rustversion",
  "tower-layer",
@@ -276,36 +366,57 @@ dependencies = [
 ]
 
 [[package]]
-name = "axum-tracing-opentelemetry"
-version = "0.14.1"
+name = "axum-core"
+version = "0.4.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "06985105829f176e9a3f113b1c71cc24e08f600ef0df4e70cd90d144f889e19f"
+checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199"
 dependencies = [
- "axum",
+ "async-trait",
+ "bytes",
+ "futures-util",
+ "http 1.1.0",
+ "http-body 1.0.1",
+ "http-body-util",
+ "mime",
+ "pin-project-lite",
+ "rustversion",
+ "sync_wrapper 1.0.1",
+ "tower-layer",
+ "tower-service",
+ "tracing",
+]
+
+[[package]]
+name = "axum-tracing-opentelemetry"
+version = "0.16.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bdad298231394729042d1f155b93f9fdf0b5ee1aea0b62404c4d7341f7d8fe08"
+dependencies = [
+ "axum 0.7.7",
  "futures-core",
  "futures-util",
- "http",
- "opentelemetry",
+ "http 1.1.0",
+ "opentelemetry 0.21.0",
  "pin-project-lite",
- "tower",
+ "tower 0.4.13",
  "tracing",
- "tracing-opentelemetry",
+ "tracing-opentelemetry 0.22.0",
  "tracing-opentelemetry-instrumentation-sdk",
 ]
 
 [[package]]
 name = "backtrace"
-version = "0.3.73"
+version = "0.3.74"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a"
+checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a"
 dependencies = [
  "addr2line",
- "cc",
  "cfg-if",
  "libc",
- "miniz_oxide",
+ "miniz_oxide 0.8.0",
  "object",
  "rustc-demangle",
+ "windows-targets 0.52.6",
 ]
 
 [[package]]
@@ -326,6 +437,29 @@ version = "0.22.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
 
+[[package]]
+name = "bindgen"
+version = "0.69.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0"
+dependencies = [
+ "bitflags 2.6.0",
+ "cexpr",
+ "clang-sys",
+ "itertools 0.12.1",
+ "lazy_static",
+ "lazycell",
+ "log",
+ "prettyplease",
+ "proc-macro2",
+ "quote",
+ "regex",
+ "rustc-hash",
+ "shlex",
+ "syn 2.0.79",
+ "which",
+]
+
 [[package]]
 name = "bit-set"
 version = "0.5.3"
@@ -355,15 +489,15 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 
 [[package]]
 name = "bitflags"
-version = "2.5.0"
+version = "2.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1"
+checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de"
 
 [[package]]
 name = "bitstream-io"
-version = "2.3.0"
+version = "2.5.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7c12d1856e42f0d817a835fe55853957c85c8c8a470114029143d3f12671446e"
+checksum = "b81e1519b0d82120d2fd469d5bfb2919a9361c48b02d82d04befc1cdd2002452"
 
 [[package]]
 name = "block-buffer"
@@ -376,9 +510,9 @@ dependencies = [
 
 [[package]]
 name = "built"
-version = "0.7.3"
+version = "0.7.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c6a6c0b39c38fd754ac338b00a88066436389c0f029da5d37d1e01091d9b7c17"
+checksum = "236e6289eda5a812bc6b53c3b024039382a2895fbbeef2d748b2931546d392c4"
 
 [[package]]
 name = "bumpalo"
@@ -394,9 +528,9 @@ checksum = "5ce89b21cab1437276d2650d57e971f9d548a2d9037cc231abdc0562b97498ce"
 
 [[package]]
 name = "bytemuck"
-version = "1.16.0"
+version = "1.18.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "78834c15cb5d5efe3452d58b1e8ba890dd62d21907f867f383358198e56ebca5"
+checksum = "94bbb0ad554ad961ddc5da507a12a29b14e4ae5bda06b19f575a3e6079d2e2ae"
 
 [[package]]
 name = "byteorder"
@@ -412,15 +546,15 @@ checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495"
 
 [[package]]
 name = "bytes"
-version = "1.6.0"
+version = "1.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9"
+checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3"
 
 [[package]]
 name = "camino"
-version = "1.1.7"
+version = "1.1.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e0ec6b951b160caa93cc0c7b209e5a3bff7aae9062213451ac99493cd844c239"
+checksum = "8b96ec4966b5813e2c0507c1f86115c8c5abaadc3980879c3424042a02fd1ad3"
 dependencies = [
  "serde",
 ]
@@ -455,14 +589,38 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "df8670b8c7b9dae1793364eafadf7239c40d669904660c5960d74cfd80b46a53"
 
 [[package]]
-name = "cc"
-version = "1.0.99"
+name = "cast"
+version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "96c51067fd44124faa7f870b4b1c969379ad32b2ba805aa959430ceaa384f695"
+checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
+
+[[package]]
+name = "castaway"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0abae9be0aaf9ea96a3b1b8b1b55c602ca751eba1b1500220cea4ecbafe7c0d5"
+dependencies = [
+ "rustversion",
+]
+
+[[package]]
+name = "cc"
+version = "1.1.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9540e661f81799159abee814118cc139a2004b3a3aa3ea37724a1b66530b90e0"
 dependencies = [
  "jobserver",
  "libc",
- "once_cell",
+ "shlex",
+]
+
+[[package]]
+name = "cexpr"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766"
+dependencies = [
+ "nom",
 ]
 
 [[package]]
@@ -488,10 +646,38 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e"
 
 [[package]]
-name = "clap"
-version = "4.5.7"
+name = "cfg_aliases"
+version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5db83dced34638ad474f39f250d7fea9598bdd239eaced1bdf45d597da0f433f"
+checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
+
+[[package]]
+name = "clang-sys"
+version = "1.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4"
+dependencies = [
+ "glob",
+ "libc",
+ "libloading",
+]
+
+[[package]]
+name = "clap"
+version = "2.34.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c"
+dependencies = [
+ "bitflags 1.3.2",
+ "textwrap",
+ "unicode-width",
+]
+
+[[package]]
+name = "clap"
+version = "4.5.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b0956a43b323ac1afaffc053ed5c4b7c1f1800bacd1683c353aabbb752515dd3"
 dependencies = [
  "clap_builder",
  "clap_derive",
@@ -499,9 +685,9 @@ dependencies = [
 
 [[package]]
 name = "clap_builder"
-version = "4.5.7"
+version = "4.5.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f7e204572485eb3fbf28f871612191521df159bc3e15a9f5064c66dba3a8c05f"
+checksum = "4d72166dd41634086d5803a47eb71ae740e61d84709c36f3c34110173db3961b"
 dependencies = [
  "anstream",
  "anstyle",
@@ -511,21 +697,40 @@ dependencies = [
 
 [[package]]
 name = "clap_derive"
-version = "4.5.5"
+version = "4.5.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c780290ccf4fb26629baa7a1081e68ced113f1d3ec302fa5948f1c381ebf06c6"
+checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab"
 dependencies = [
  "heck 0.5.0",
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.79",
 ]
 
 [[package]]
 name = "clap_lex"
-version = "0.7.1"
+version = "0.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4b82cf0babdbd58558212896d1a4272303a57bdb245c2bf1147185fb45640e70"
+checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97"
+
+[[package]]
+name = "cmake"
+version = "0.1.51"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fb1e43aa7fd152b1f968787f7dbcdeb306d1867ff373c69955211876c053f91a"
+dependencies = [
+ "cc",
+]
+
+[[package]]
+name = "codespan-reporting"
+version = "0.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e"
+dependencies = [
+ "termcolor",
+ "unicode-width",
+]
 
 [[package]]
 name = "color_quant"
@@ -535,9 +740,23 @@ checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b"
 
 [[package]]
 name = "colorchoice"
-version = "1.0.1"
+version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422"
+checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0"
+
+[[package]]
+name = "compact_str"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6050c3a16ddab2e412160b31f2c871015704239bca62f72f6e5f0be631d3f644"
+dependencies = [
+ "castaway",
+ "cfg-if",
+ "itoa",
+ "rustversion",
+ "ryu",
+ "static_assertions",
+]
 
 [[package]]
 name = "console"
@@ -564,15 +783,15 @@ dependencies = [
 
 [[package]]
 name = "core-foundation-sys"
-version = "0.8.6"
+version = "0.8.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f"
+checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
 
 [[package]]
 name = "cpufeatures"
-version = "0.2.12"
+version = "0.2.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504"
+checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0"
 dependencies = [
  "libc",
 ]
@@ -586,6 +805,42 @@ dependencies = [
  "cfg-if",
 ]
 
+[[package]]
+name = "criterion"
+version = "0.3.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b01d6de93b2b6c65e17c634a26653a29d107b3c98c607c765bf38d041531cd8f"
+dependencies = [
+ "atty",
+ "cast",
+ "clap 2.34.0",
+ "criterion-plot",
+ "csv",
+ "itertools 0.10.5",
+ "lazy_static",
+ "num-traits",
+ "oorandom",
+ "plotters",
+ "rayon",
+ "regex",
+ "serde",
+ "serde_cbor",
+ "serde_derive",
+ "serde_json",
+ "tinytemplate",
+ "walkdir",
+]
+
+[[package]]
+name = "criterion-plot"
+version = "0.4.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2673cc8207403546f45f5fd319a974b1e6983ad1a3ee7e6041650013be041876"
+dependencies = [
+ "cast",
+ "itertools 0.10.5",
+]
+
 [[package]]
 name = "crossbeam-channel"
 version = "0.5.13"
@@ -622,15 +877,15 @@ checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80"
 
 [[package]]
 name = "crossterm"
-version = "0.27.0"
+version = "0.28.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f476fe445d41c9e991fd07515a6f463074b782242ccf4a5b7b1d1012e70824df"
+checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6"
 dependencies = [
- "bitflags 2.5.0",
+ "bitflags 2.6.0",
  "crossterm_winapi",
- "libc",
  "mio",
  "parking_lot",
+ "rustix",
  "signal-hook",
  "signal-hook-mio",
  "winapi",
@@ -662,20 +917,85 @@ dependencies = [
 ]
 
 [[package]]
-name = "ctrlc"
-version = "3.4.4"
+name = "csv"
+version = "1.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "672465ae37dc1bc6380a6547a8883d5dd397b0f1faaad4f265726cc7042a5345"
+checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe"
 dependencies = [
- "nix",
- "windows-sys 0.52.0",
+ "csv-core",
+ "itoa",
+ "ryu",
+ "serde",
+]
+
+[[package]]
+name = "csv-core"
+version = "0.1.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
+name = "ctrlc"
+version = "3.4.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "90eeab0aa92f3f9b4e87f258c72b139c207d251f9cbc1080a0086b86a8870dd3"
+dependencies = [
+ "nix 0.29.0",
+ "windows-sys 0.59.0",
+]
+
+[[package]]
+name = "cxx"
+version = "1.0.128"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "54ccead7d199d584d139148b04b4a368d1ec7556a1d9ea2548febb1b9d49f9a4"
+dependencies = [
+ "cc",
+ "cxxbridge-flags",
+ "cxxbridge-macro",
+ "link-cplusplus",
+]
+
+[[package]]
+name = "cxx-build"
+version = "1.0.128"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c77953e99f01508f89f55c494bfa867171ef3a6c8cea03d26975368f2121a5c1"
+dependencies = [
+ "cc",
+ "codespan-reporting",
+ "once_cell",
+ "proc-macro2",
+ "quote",
+ "scratch",
+ "syn 2.0.79",
+]
+
+[[package]]
+name = "cxxbridge-flags"
+version = "1.0.128"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "65777e06cc48f0cb0152024c77d6cf9e4bdb4408e7b48bea993d42fa0f5b02b6"
+
+[[package]]
+name = "cxxbridge-macro"
+version = "1.0.128"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "98532a60dedaebc4848cb2cba5023337cc9ea3af16a5b062633fabfd9f18fb60"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.79",
 ]
 
 [[package]]
 name = "darling"
-version = "0.20.9"
+version = "0.20.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "83b2eb4d90d12bdda5ed17de686c2acb4c57914f8f921b8da7e112b5a36f3fe1"
+checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989"
 dependencies = [
  "darling_core",
  "darling_macro",
@@ -683,27 +1003,27 @@ dependencies = [
 
 [[package]]
 name = "darling_core"
-version = "0.20.9"
+version = "0.20.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "622687fe0bac72a04e5599029151f5796111b90f1baaa9b544d807a5e31cd120"
+checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5"
 dependencies = [
  "fnv",
  "ident_case",
  "proc-macro2",
  "quote",
  "strsim",
- "syn 2.0.66",
+ "syn 2.0.79",
 ]
 
 [[package]]
 name = "darling_macro"
-version = "0.20.9"
+version = "0.20.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "733cabb43482b1a1b53eee8583c2b9e8684d592215ea83efd305dd31bc2f0178"
+checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806"
 dependencies = [
  "darling_core",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -717,33 +1037,33 @@ dependencies = [
 
 [[package]]
 name = "derive_builder"
-version = "0.20.0"
+version = "0.20.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0350b5cb0331628a5916d6c5c0b72e97393b8b6b03b47a9284f4e7f5a405ffd7"
+checksum = "cd33f37ee6a119146a1781d3356a7c26028f83d779b2e04ecd45fdc75c76877b"
 dependencies = [
  "derive_builder_macro",
 ]
 
 [[package]]
 name = "derive_builder_core"
-version = "0.20.0"
+version = "0.20.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d48cda787f839151732d396ac69e3473923d54312c070ee21e9effcaa8ca0b1d"
+checksum = "7431fa049613920234f22c47fdc33e6cf3ee83067091ea4277a3f8c4587aae38"
 dependencies = [
  "darling",
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.79",
 ]
 
 [[package]]
 name = "derive_builder_macro"
-version = "0.20.0"
+version = "0.20.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "206868b8242f27cecce124c19fd88157fbd0dd334df2587f36417bafbc85097b"
+checksum = "4abae7035bf79b9877b779505d8cf3749285b80c43941eda66604841889451dc"
 dependencies = [
  "derive_builder_core",
- "syn 2.0.66",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -756,33 +1076,13 @@ dependencies = [
  "crypto-common",
 ]
 
-[[package]]
-name = "dirs"
-version = "4.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059"
-dependencies = [
- "dirs-sys 0.3.7",
-]
-
 [[package]]
 name = "dirs"
 version = "5.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225"
 dependencies = [
- "dirs-sys 0.4.1",
-]
-
-[[package]]
-name = "dirs-sys"
-version = "0.3.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6"
-dependencies = [
- "libc",
- "redox_users",
- "winapi",
+ "dirs-sys",
 ]
 
 [[package]]
@@ -798,15 +1098,10 @@ dependencies = [
 ]
 
 [[package]]
-name = "displaydoc"
-version = "0.2.4"
+name = "dunce"
+version = "1.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.66",
-]
+checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813"
 
 [[package]]
 name = "easy-cast"
@@ -819,9 +1114,9 @@ dependencies = [
 
 [[package]]
 name = "either"
-version = "1.12.0"
+version = "1.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3dca9240753cf90908d7e4aac30f630662b02aebaa1b58a3cadabdb23385b58b"
+checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0"
 
 [[package]]
 name = "encode_unicode"
@@ -871,9 +1166,9 @@ checksum = "887d93f60543e9a9362ef8a21beedd0a833c5d9610e18c67abe15a5963dcb1a4"
 dependencies = [
  "bit_field",
  "flume",
- "half",
+ "half 2.4.1",
  "lebe",
- "miniz_oxide",
+ "miniz_oxide 0.7.4",
  "rayon-core",
  "smallvec",
  "zune-inflate",
@@ -891,15 +1186,15 @@ dependencies = [
 
 [[package]]
 name = "fastrand"
-version = "2.1.0"
+version = "2.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a"
+checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6"
 
 [[package]]
 name = "fdeflate"
-version = "0.3.4"
+version = "0.3.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4f9bfee30e4dedf0ab8b422f03af778d9612b63f502710fc500a334ebe2de645"
+checksum = "d8090f921a24b04994d9929e204f50b498a33ea6ba559ffaa05e04f7ee7fb5ab"
 dependencies = [
  "simd-adler32",
 ]
@@ -912,12 +1207,12 @@ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80"
 
 [[package]]
 name = "flate2"
-version = "1.0.30"
+version = "1.0.34"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5f54427cfd1c7829e2a139fcefea601bf088ebca651d2bf53ebc600eac295dae"
+checksum = "a1b589b4dc103969ad3cf85c950899926ec64300a1a46d76c03a6072957036f0"
 dependencies = [
  "crc32fast",
- "miniz_oxide",
+ "miniz_oxide 0.8.0",
 ]
 
 [[package]]
@@ -981,6 +1276,12 @@ dependencies = [
  "num",
 ]
 
+[[package]]
+name = "fs_extra"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
+
 [[package]]
 name = "futures"
 version = "0.3.30"
@@ -1037,7 +1338,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -1114,18 +1415,24 @@ dependencies = [
 
 [[package]]
 name = "gimli"
-version = "0.29.0"
+version = "0.31.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd"
+checksum = "32085ea23f3234fc7846555e85283ba4de91e21016dc0455a16286d87a292d64"
+
+[[package]]
+name = "glob"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"
 
 [[package]]
 name = "grpc-metadata"
 version = "0.1.0"
 dependencies = [
- "opentelemetry",
+ "opentelemetry 0.20.0",
  "tonic 0.10.2",
  "tracing",
- "tracing-opentelemetry",
+ "tracing-opentelemetry 0.21.0",
 ]
 
 [[package]]
@@ -1139,14 +1446,39 @@ dependencies = [
  "futures-core",
  "futures-sink",
  "futures-util",
- "http",
- "indexmap 2.2.6",
+ "http 0.2.12",
+ "indexmap 2.5.0",
  "slab",
  "tokio",
  "tokio-util",
  "tracing",
 ]
 
+[[package]]
+name = "h2"
+version = "0.4.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "524e8ac6999421f49a846c2d4411f337e53497d8ec55d67753beffa43c5d9205"
+dependencies = [
+ "atomic-waker",
+ "bytes",
+ "fnv",
+ "futures-core",
+ "futures-sink",
+ "http 1.1.0",
+ "indexmap 2.5.0",
+ "slab",
+ "tokio",
+ "tokio-util",
+ "tracing",
+]
+
+[[package]]
+name = "half"
+version = "1.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1b43ede17f21864e81be2fa654110bf1e793774238d86ef8555c37e6519c0403"
+
 [[package]]
 name = "half"
 version = "2.4.1"
@@ -1163,20 +1495,15 @@ version = "0.12.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
 
-[[package]]
-name = "hashbrown"
-version = "0.13.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "33ff8ae62cd3a9102e5637afc8452c55acf3844001bd5374e0b0bd7b6616c038"
-dependencies = [
- "ahash",
-]
-
 [[package]]
 name = "hashbrown"
 version = "0.14.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
+dependencies = [
+ "ahash",
+ "allocator-api2",
+]
 
 [[package]]
 name = "heck"
@@ -1190,6 +1517,15 @@ version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
 
+[[package]]
+name = "hermit-abi"
+version = "0.1.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "hermit-abi"
 version = "0.3.9"
@@ -1202,7 +1538,7 @@ version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2b780635574b3d92f036890d8373433d6f9fc7abb320ee42a5c25897fc8ed732"
 dependencies = [
- "dirs 5.0.1",
+ "dirs",
  "futures",
  "indicatif",
  "log",
@@ -1217,6 +1553,15 @@ dependencies = [
  "ureq",
 ]
 
+[[package]]
+name = "home"
+version = "0.5.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5"
+dependencies = [
+ "windows-sys 0.52.0",
+]
+
 [[package]]
 name = "hostname"
 version = "0.3.1"
@@ -1239,6 +1584,17 @@ dependencies = [
  "itoa",
 ]
 
+[[package]]
+name = "http"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258"
+dependencies = [
+ "bytes",
+ "fnv",
+ "itoa",
+]
+
 [[package]]
 name = "http-body"
 version = "0.4.6"
@@ -1246,21 +1602,38 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2"
 dependencies = [
  "bytes",
- "http",
+ "http 0.2.12",
  "pin-project-lite",
 ]
 
 [[package]]
-name = "http-range-header"
-version = "0.3.1"
+name = "http-body"
+version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "add0ab9360ddbd88cfeb3bd9574a1d85cfdfa14db10b3e21d3700dbc4328758f"
+checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184"
+dependencies = [
+ "bytes",
+ "http 1.1.0",
+]
+
+[[package]]
+name = "http-body-util"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f"
+dependencies = [
+ "bytes",
+ "futures-util",
+ "http 1.1.0",
+ "http-body 1.0.1",
+ "pin-project-lite",
+]
 
 [[package]]
 name = "httparse"
-version = "1.9.3"
+version = "1.9.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d0e7a4dd27b9476dc40cb050d3632d3bba3a70ddbff012285f7f8559a1e7e545"
+checksum = "0fcc0b4a115bf80b728eb8ea024ad5bd707b615bfed49e0665b6e0f86fd082d9"
 
 [[package]]
 name = "httpdate"
@@ -1270,17 +1643,17 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
 
 [[package]]
 name = "hyper"
-version = "0.14.29"
+version = "0.14.30"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f361cde2f109281a220d4307746cdfd5ee3f410da58a70377762396775634b33"
+checksum = "a152ddd61dfaec7273fe8419ab357f33aee0d914c5f4efbf0d96fa749eea5ec9"
 dependencies = [
  "bytes",
  "futures-channel",
  "futures-core",
  "futures-util",
- "h2",
- "http",
- "http-body",
+ "h2 0.3.26",
+ "http 0.2.12",
+ "http-body 0.4.6",
  "httparse",
  "httpdate",
  "itoa",
@@ -1292,13 +1665,53 @@ dependencies = [
  "want",
 ]
 
+[[package]]
+name = "hyper"
+version = "1.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05"
+dependencies = [
+ "bytes",
+ "futures-channel",
+ "futures-util",
+ "h2 0.4.6",
+ "http 1.1.0",
+ "http-body 1.0.1",
+ "httparse",
+ "httpdate",
+ "itoa",
+ "pin-project-lite",
+ "smallvec",
+ "tokio",
+ "want",
+]
+
+[[package]]
+name = "hyper-rustls"
+version = "0.27.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333"
+dependencies = [
+ "futures-util",
+ "http 1.1.0",
+ "hyper 1.4.1",
+ "hyper-util",
+ "log",
+ "rustls 0.23.13",
+ "rustls-native-certs",
+ "rustls-pki-types",
+ "tokio",
+ "tokio-rustls",
+ "tower-service",
+]
+
 [[package]]
 name = "hyper-timeout"
 version = "0.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1"
 dependencies = [
- "hyper",
+ "hyper 0.14.30",
  "pin-project-lite",
  "tokio",
  "tokio-io-timeout",
@@ -1311,128 +1724,29 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905"
 dependencies = [
  "bytes",
- "hyper",
+ "hyper 0.14.30",
  "native-tls",
  "tokio",
  "tokio-native-tls",
 ]
 
 [[package]]
-name = "icu_collections"
-version = "1.5.0"
+name = "hyper-util"
+version = "0.1.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526"
+checksum = "41296eb09f183ac68eec06e03cdbea2e759633d4067b2f6552fc2e009bcad08b"
 dependencies = [
- "displaydoc",
- "yoke",
- "zerofrom",
- "zerovec",
-]
-
-[[package]]
-name = "icu_locid"
-version = "1.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637"
-dependencies = [
- "displaydoc",
- "litemap",
- "tinystr",
- "writeable",
- "zerovec",
-]
-
-[[package]]
-name = "icu_locid_transform"
-version = "1.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e"
-dependencies = [
- "displaydoc",
- "icu_locid",
- "icu_locid_transform_data",
- "icu_provider",
- "tinystr",
- "zerovec",
-]
-
-[[package]]
-name = "icu_locid_transform_data"
-version = "1.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e"
-
-[[package]]
-name = "icu_normalizer"
-version = "1.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f"
-dependencies = [
- "displaydoc",
- "icu_collections",
- "icu_normalizer_data",
- "icu_properties",
- "icu_provider",
- "smallvec",
- "utf16_iter",
- "utf8_iter",
- "write16",
- "zerovec",
-]
-
-[[package]]
-name = "icu_normalizer_data"
-version = "1.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516"
-
-[[package]]
-name = "icu_properties"
-version = "1.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1f8ac670d7422d7f76b32e17a5db556510825b29ec9154f235977c9caba61036"
-dependencies = [
- "displaydoc",
- "icu_collections",
- "icu_locid_transform",
- "icu_properties_data",
- "icu_provider",
- "tinystr",
- "zerovec",
-]
-
-[[package]]
-name = "icu_properties_data"
-version = "1.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569"
-
-[[package]]
-name = "icu_provider"
-version = "1.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9"
-dependencies = [
- "displaydoc",
- "icu_locid",
- "icu_provider_macros",
- "stable_deref_trait",
- "tinystr",
- "writeable",
- "yoke",
- "zerofrom",
- "zerovec",
-]
-
-[[package]]
-name = "icu_provider_macros"
-version = "1.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.66",
+ "bytes",
+ "futures-channel",
+ "futures-util",
+ "http 1.1.0",
+ "http-body 1.0.1",
+ "hyper 1.4.1",
+ "pin-project-lite",
+ "socket2",
+ "tokio",
+ "tower-service",
+ "tracing",
 ]
 
 [[package]]
@@ -1443,24 +1757,22 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
 
 [[package]]
 name = "idna"
-version = "1.0.0"
+version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4716a3a0933a1d01c2f72450e89596eb51dd34ef3c211ccd875acdf1f8fe47ed"
+checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6"
 dependencies = [
- "icu_normalizer",
- "icu_properties",
- "smallvec",
- "utf8_iter",
+ "unicode-bidi",
+ "unicode-normalization",
 ]
 
 [[package]]
 name = "image"
-version = "0.25.1"
+version = "0.25.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fd54d660e773627692c524beaad361aca785a4f9f5730ce91f42aabe5bce3d11"
+checksum = "99314c8a2152b8ddb211f924cdae532d8c5e4c8bb54728e12fff1b0cd5963a10"
 dependencies = [
  "bytemuck",
- "byteorder",
+ "byteorder-lite",
  "color_quant",
  "exr",
  "gif",
@@ -1478,12 +1790,12 @@ dependencies = [
 
 [[package]]
 name = "image-webp"
-version = "0.1.2"
+version = "0.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d730b085583c4d789dfd07fdcf185be59501666a90c97c40162b37e4fdad272d"
+checksum = "f79afb8cbee2ef20f59ccd477a218c12a93943d075b492015ecb1bb81f8ee904"
 dependencies = [
  "byteorder-lite",
- "thiserror",
+ "quick-error",
 ]
 
 [[package]]
@@ -1504,9 +1816,9 @@ dependencies = [
 
 [[package]]
 name = "indexmap"
-version = "2.2.6"
+version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26"
+checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5"
 dependencies = [
  "equivalent",
  "hashbrown 0.14.5",
@@ -1538,11 +1850,21 @@ version = "0.14.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "94bd26b1b737bc11f183620072e188d1c6ede67e0e78682228d66b49ec510e17"
 dependencies = [
- "opentelemetry",
+ "opentelemetry 0.20.0",
  "opentelemetry-otlp",
  "thiserror",
  "tracing",
- "tracing-opentelemetry",
+ "tracing-opentelemetry 0.21.0",
+]
+
+[[package]]
+name = "instability"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b23a0c8dfe501baac4adf6ebbfa6eddf8f0c07f56b058cc1288017e32397846c"
+dependencies = [
+ "quote",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -1562,20 +1884,20 @@ checksum = "c34819042dc3d3971c46c2190835914dfbe0c3c13f61449b2997f4e9722dfa60"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.79",
 ]
 
 [[package]]
 name = "ipnet"
-version = "2.9.0"
+version = "2.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3"
+checksum = "187674a687eed5fe42285b40c6291f9a01517d415fad1c3cbc6a9f778af7fcd4"
 
 [[package]]
 name = "is_terminal_polyfill"
-version = "1.70.0"
+version = "1.70.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800"
+checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf"
 
 [[package]]
 name = "iso8601"
@@ -1613,6 +1935,15 @@ dependencies = [
  "either",
 ]
 
+[[package]]
+name = "itertools"
+version = "0.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186"
+dependencies = [
+ "either",
+]
+
 [[package]]
 name = "itoa"
 version = "1.0.11"
@@ -1621,9 +1952,9 @@ checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b"
 
 [[package]]
 name = "jobserver"
-version = "0.1.31"
+version = "0.1.32"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d2b099aaa34a9751c5bf0878add70444e1ed2dd73f347be99003d4577277de6e"
+checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0"
 dependencies = [
  "libc",
 ]
@@ -1636,9 +1967,9 @@ checksum = "f5d4a7da358eff58addd2877a45865158f0d78c911d43a5784ceb7bbf52833b0"
 
 [[package]]
 name = "js-sys"
-version = "0.3.69"
+version = "0.3.70"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d"
+checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a"
 dependencies = [
  "wasm-bindgen",
 ]
@@ -1653,7 +1984,7 @@ dependencies = [
  "anyhow",
  "base64 0.21.7",
  "bytecount",
- "clap",
+ "clap 4.5.18",
  "fancy-regex",
  "fraction",
  "getrandom",
@@ -1675,9 +2006,15 @@ dependencies = [
 
 [[package]]
 name = "lazy_static"
-version = "1.4.0"
+version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
+checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
+
+[[package]]
+name = "lazycell"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
 
 [[package]]
 name = "lebe"
@@ -1687,9 +2024,9 @@ checksum = "03087c2bad5e1034e8cace5926dec053fb3790248370865f5117a7d0213354c8"
 
 [[package]]
 name = "libc"
-version = "0.2.155"
+version = "0.2.159"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c"
+checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5"
 
 [[package]]
 name = "libfuzzer-sys"
@@ -1702,6 +2039,16 @@ dependencies = [
  "once_cell",
 ]
 
+[[package]]
+name = "libloading"
+version = "0.8.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4"
+dependencies = [
+ "cfg-if",
+ "windows-targets 0.52.6",
+]
+
 [[package]]
 name = "libm"
 version = "0.2.8"
@@ -1714,22 +2061,25 @@ version = "0.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d"
 dependencies = [
- "bitflags 2.5.0",
+ "bitflags 2.6.0",
  "libc",
 ]
 
+[[package]]
+name = "link-cplusplus"
+version = "1.0.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9d240c6f7e1ba3a28b0249f774e6a9dd0175054b52dfbb61b16eb8505c3785c9"
+dependencies = [
+ "cc",
+]
+
 [[package]]
 name = "linux-raw-sys"
 version = "0.4.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89"
 
-[[package]]
-name = "litemap"
-version = "0.7.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704"
-
 [[package]]
 name = "lock_api"
 version = "0.4.12"
@@ -1742,9 +2092,9 @@ dependencies = [
 
 [[package]]
 name = "log"
-version = "0.4.21"
+version = "0.4.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c"
+checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24"
 
 [[package]]
 name = "loop9"
@@ -1756,12 +2106,12 @@ dependencies = [
 ]
 
 [[package]]
-name = "mach2"
-version = "0.4.2"
+name = "lru"
+version = "0.12.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "19b955cdeb2a02b9117f121ce63aa52d08ade45de53e48fe6a38b39c10f6f709"
+checksum = "37ee39891760e7d94734f6f63fedc29a2e4a152f836120753a72503f09fcf904"
 dependencies = [
- "libc",
+ "hashbrown 0.14.5",
 ]
 
 [[package]]
@@ -1808,7 +2158,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8ea1f30cedd69f0a2954655f7188c6a834246d2bcf1e315e2ac40c4b24dc9519"
 dependencies = [
  "cfg-if",
- "rayon",
 ]
 
 [[package]]
@@ -1818,25 +2167,36 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
 
 [[package]]
-name = "metrics"
-version = "0.21.1"
+name = "memoffset"
+version = "0.9.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fde3af1a009ed76a778cb84fdef9e7dbbdf5775ae3e4cc1f434a6a307f6f76c5"
+checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a"
+dependencies = [
+ "autocfg",
+]
+
+[[package]]
+name = "metrics"
+version = "0.23.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "884adb57038347dfbaf2d5065887b6cf4312330dc8e94bc30a1a839bd79d3261"
 dependencies = [
  "ahash",
- "metrics-macros",
  "portable-atomic",
 ]
 
 [[package]]
 name = "metrics-exporter-prometheus"
-version = "0.12.2"
+version = "0.15.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1d4fa7ce7c4862db464a37b0b31d89bca874562f034bd7993895572783d02950"
+checksum = "b4f0c8427b39666bf970460908b213ec09b3b350f20c0c2eabcbba51704a08e6"
 dependencies = [
- "base64 0.21.7",
- "hyper",
- "indexmap 1.9.3",
+ "base64 0.22.1",
+ "http-body-util",
+ "hyper 1.4.1",
+ "hyper-rustls",
+ "hyper-util",
+ "indexmap 2.5.0",
  "ipnet",
  "metrics",
  "metrics-util",
@@ -1846,26 +2206,15 @@ dependencies = [
  "tracing",
 ]
 
-[[package]]
-name = "metrics-macros"
-version = "0.7.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "38b4faf00617defe497754acde3024865bc143d44a86799b24e191ecff91354f"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.66",
-]
-
 [[package]]
 name = "metrics-util"
-version = "0.15.1"
+version = "0.17.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4de2ed6e491ed114b40b732e4d1659a9d53992ebd87490c44a6ffe23739d973e"
+checksum = "4259040465c955f9f2f1a4a8a16dc46726169bca0f88e8fb2dbeced487c3e828"
 dependencies = [
  "crossbeam-epoch",
  "crossbeam-utils",
- "hashbrown 0.13.1",
+ "hashbrown 0.14.5",
  "metrics",
  "num_cpus",
  "quanta",
@@ -1880,9 +2229,9 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
 
 [[package]]
 name = "mime_guess"
-version = "2.0.4"
+version = "2.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4192263c238a5f0d0c6bfd21f336a313a4ce1c450542449ca191bb657b4642ef"
+checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e"
 dependencies = [
  "mime",
  "unicase",
@@ -1890,10 +2239,22 @@ dependencies = [
 
 [[package]]
 name = "minijinja"
-version = "1.0.12"
-source = "git+https://github.com/mitsuhiko/minijinja.git?rev=5cd4efb#5cd4efb9e2639247df275fe6e22a5dbe0ce71b28"
+version = "2.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1028b628753a7e1a88fc59c9ba4b02ecc3bc0bd3c7af23df667bc28df9b3310e"
 dependencies = [
  "serde",
+ "serde_json",
+]
+
+[[package]]
+name = "minijinja-contrib"
+version = "2.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "39ffd46ee854be23604a20efd6c9655374fefbe4d44b949dc0f907305d92873a"
+dependencies = [
+ "minijinja",
+ "serde",
 ]
 
 [[package]]
@@ -1904,26 +2265,42 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
 
 [[package]]
 name = "miniz_oxide"
-version = "0.7.3"
+version = "0.7.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87dfd01fe195c66b572b37921ad8803d010623c0aca821bea2302239d155cdae"
+checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08"
 dependencies = [
  "adler",
+]
+
+[[package]]
+name = "miniz_oxide"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1"
+dependencies = [
+ "adler2",
  "simd-adler32",
 ]
 
 [[package]]
 name = "mio"
-version = "0.8.11"
+version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c"
+checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec"
 dependencies = [
+ "hermit-abi 0.3.9",
  "libc",
  "log",
  "wasi",
- "windows-sys 0.48.0",
+ "windows-sys 0.52.0",
 ]
 
+[[package]]
+name = "mirai-annotations"
+version = "1.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c9be0862c1b3f26a88803c4a49de6889c10e608b3ee9344e6ef5b45fb37ad3d1"
+
 [[package]]
 name = "monostate"
 version = "0.1.13"
@@ -1942,7 +2319,7 @@ checksum = "a7ce64b975ed4f123575d11afd9491f2e37bbd5813fbfbc0f09ae1fbddea74e0"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -2003,17 +2380,17 @@ dependencies = [
  "async-rustls",
  "async-trait",
  "awaitdrop",
- "axum",
+ "axum 0.6.20",
  "base64 0.13.1",
  "bytes",
  "futures",
  "hostname",
- "hyper",
+ "hyper 0.14.30",
  "muxado",
  "once_cell",
  "parking_lot",
  "regex",
- "rustls-pemfile",
+ "rustls-pemfile 1.0.4",
  "serde",
  "serde_json",
  "thiserror",
@@ -2030,9 +2407,21 @@ version = "0.28.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4"
 dependencies = [
- "bitflags 2.5.0",
+ "bitflags 2.6.0",
  "cfg-if",
- "cfg_aliases",
+ "cfg_aliases 0.1.1",
+ "libc",
+]
+
+[[package]]
+name = "nix"
+version = "0.29.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46"
+dependencies = [
+ "bitflags 2.6.0",
+ "cfg-if",
+ "cfg_aliases 0.2.1",
  "libc",
 ]
 
@@ -2093,9 +2482,9 @@ dependencies = [
 
 [[package]]
 name = "num-bigint"
-version = "0.4.5"
+version = "0.4.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c165a9ab64cf766f73521c0dd2cfdff64f488b8f0b3e621face3462d3db536d7"
+checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9"
 dependencies = [
  "num-integer",
  "num-traits",
@@ -2130,7 +2519,7 @@ checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -2180,7 +2569,7 @@ version = "1.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43"
 dependencies = [
- "hermit-abi",
+ "hermit-abi 0.3.9",
  "libc",
 ]
 
@@ -2201,18 +2590,21 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
 
 [[package]]
 name = "object"
-version = "0.36.0"
+version = "0.36.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "576dfe1fc8f9df304abb159d767a29d0476f7750fbf8aa7ad07816004a207434"
+checksum = "084f1a5821ac4c651660a94a7153d27ac9d8a53736203f58b31945ded098070a"
 dependencies = [
  "memchr",
 ]
 
 [[package]]
 name = "once_cell"
-version = "1.19.0"
+version = "1.20.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
+checksum = "82881c4be219ab5faaf2ad5e5e5ecdff8c66bd7402ca3160975c93b24961afd1"
+dependencies = [
+ "portable-atomic",
+]
 
 [[package]]
 name = "onig"
@@ -2237,12 +2629,18 @@ dependencies = [
 ]
 
 [[package]]
-name = "openssl"
-version = "0.10.64"
+name = "oorandom"
+version = "11.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f"
+checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9"
+
+[[package]]
+name = "openssl"
+version = "0.10.66"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9529f4786b70a3e8c61e11179af17ab6188ad8d0ded78c5529441ed39d4bd9c1"
 dependencies = [
- "bitflags 2.5.0",
+ "bitflags 2.6.0",
  "cfg-if",
  "foreign-types",
  "libc",
@@ -2259,7 +2657,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -2270,9 +2668,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
 
 [[package]]
 name = "openssl-sys"
-version = "0.9.102"
+version = "0.9.103"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c597637d56fbc83893a35eb0dd04b2b8e7a50c91e64e9493e398b5df4fb45fa2"
+checksum = "7f9e8deee91df40a943c71b917e5874b951d32a802526c85721ce3b776c929d6"
 dependencies = [
  "cc",
  "libc",
@@ -2287,19 +2685,37 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9591d937bc0e6d2feb6f71a559540ab300ea49955229c347a517a28d27784c54"
 dependencies = [
  "opentelemetry_api",
- "opentelemetry_sdk",
+ "opentelemetry_sdk 0.20.0",
 ]
 
 [[package]]
-name = "opentelemetry-http"
-version = "0.9.0"
+name = "opentelemetry"
+version = "0.21.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c7594ec0e11d8e33faf03530a4c49af7064ebba81c1480e01be67d90b356508b"
+checksum = "1e32339a5dc40459130b3bd269e9892439f55b33e772d2a9d402a789baaf4e8a"
 dependencies = [
- "async-trait",
- "bytes",
- "http",
- "opentelemetry_api",
+ "futures-core",
+ "futures-sink",
+ "indexmap 2.5.0",
+ "js-sys",
+ "once_cell",
+ "pin-project-lite",
+ "thiserror",
+ "urlencoding",
+]
+
+[[package]]
+name = "opentelemetry"
+version = "0.23.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1b69a91d4893e713e06f724597ad630f1fa76057a5e1026c0ca67054a9032a76"
+dependencies = [
+ "futures-core",
+ "futures-sink",
+ "js-sys",
+ "once_cell",
+ "pin-project-lite",
+ "thiserror",
 ]
 
 [[package]]
@@ -2310,11 +2726,11 @@ checksum = "7e5e5a5c4135864099f3faafbe939eb4d7f9b80ebf68a8448da961b32a7c1275"
 dependencies = [
  "async-trait",
  "futures-core",
- "http",
+ "http 0.2.12",
  "opentelemetry-proto",
  "opentelemetry-semantic-conventions",
  "opentelemetry_api",
- "opentelemetry_sdk",
+ "opentelemetry_sdk 0.20.0",
  "prost 0.11.9",
  "thiserror",
  "tokio",
@@ -2328,7 +2744,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b1e3f814aa9f8c905d0ee4bde026afd3b2577a97c10e1699912e3e44f0c4cbeb"
 dependencies = [
  "opentelemetry_api",
- "opentelemetry_sdk",
+ "opentelemetry_sdk 0.20.0",
  "prost 0.11.9",
  "tonic 0.9.2",
 ]
@@ -2339,7 +2755,7 @@ version = "0.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "73c9f9340ad135068800e7f1b24e9e09ed9e7143f5bf8518ded3d3ec69789269"
 dependencies = [
- "opentelemetry",
+ "opentelemetry 0.20.0",
 ]
 
 [[package]]
@@ -2371,7 +2787,7 @@ dependencies = [
  "futures-util",
  "once_cell",
  "opentelemetry_api",
- "ordered-float",
+ "ordered-float 3.9.2",
  "percent-encoding",
  "rand",
  "regex",
@@ -2381,6 +2797,46 @@ dependencies = [
  "tokio-stream",
 ]
 
+[[package]]
+name = "opentelemetry_sdk"
+version = "0.21.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2f16aec8a98a457a52664d69e0091bac3a0abd18ead9b641cb00202ba4e0efe4"
+dependencies = [
+ "async-trait",
+ "crossbeam-channel",
+ "futures-channel",
+ "futures-executor",
+ "futures-util",
+ "glob",
+ "once_cell",
+ "opentelemetry 0.21.0",
+ "ordered-float 4.3.0",
+ "percent-encoding",
+ "rand",
+ "thiserror",
+]
+
+[[package]]
+name = "opentelemetry_sdk"
+version = "0.23.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ae312d58eaa90a82d2e627fd86e075cf5230b3f11794e2ed74199ebbe572d4fd"
+dependencies = [
+ "async-trait",
+ "futures-channel",
+ "futures-executor",
+ "futures-util",
+ "glob",
+ "lazy_static",
+ "once_cell",
+ "opentelemetry 0.23.0",
+ "ordered-float 4.3.0",
+ "percent-encoding",
+ "rand",
+ "thiserror",
+]
+
 [[package]]
 name = "option-ext"
 version = "0.2.0"
@@ -2396,6 +2852,15 @@ dependencies = [
  "num-traits",
 ]
 
+[[package]]
+name = "ordered-float"
+version = "4.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "44d501f1a72f71d3c063a6bbc8f7271fa73aa09fe5d6283b6571e2ed176a2537"
+dependencies = [
+ "num-traits",
+]
+
 [[package]]
 name = "overload"
 version = "0.1.1"
@@ -2433,7 +2898,7 @@ dependencies = [
  "libc",
  "redox_syscall",
  "smallvec",
- "windows-targets 0.52.5",
+ "windows-targets 0.52.6",
 ]
 
 [[package]]
@@ -2455,7 +2920,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db"
 dependencies = [
  "fixedbitset",
- "indexmap 2.2.6",
+ "indexmap 2.5.0",
 ]
 
 [[package]]
@@ -2475,7 +2940,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -2492,28 +2957,56 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
 
 [[package]]
 name = "pkg-config"
-version = "0.3.30"
+version = "0.3.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec"
+checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2"
+
+[[package]]
+name = "plotters"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747"
+dependencies = [
+ "num-traits",
+ "plotters-backend",
+ "plotters-svg",
+ "wasm-bindgen",
+ "web-sys",
+]
+
+[[package]]
+name = "plotters-backend"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a"
+
+[[package]]
+name = "plotters-svg"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670"
+dependencies = [
+ "plotters-backend",
+]
 
 [[package]]
 name = "png"
-version = "0.17.13"
+version = "0.17.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "06e4b0d3d1312775e782c86c91a111aa1f910cbb65e1337f9975b5f9a554b5e1"
+checksum = "52f9d46a34a05a6a57566bc2bfae066ef07585a6e3fa30fbbdff5936380623f0"
 dependencies = [
  "bitflags 1.3.2",
  "crc32fast",
  "fdeflate",
  "flate2",
- "miniz_oxide",
+ "miniz_oxide 0.8.0",
 ]
 
 [[package]]
 name = "portable-atomic"
-version = "1.6.0"
+version = "1.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0"
+checksum = "cc9c68a3f6da06753e9335d63e27f6b9754dd1920d941135b7ea8224f141adb2"
 
 [[package]]
 name = "powerfmt"
@@ -2523,18 +3016,21 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
 
 [[package]]
 name = "ppv-lite86"
-version = "0.2.17"
+version = "0.2.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
+checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04"
+dependencies = [
+ "zerocopy",
+]
 
 [[package]]
 name = "prettyplease"
-version = "0.2.20"
+version = "0.2.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e"
+checksum = "479cf940fbbb3426c32c5d5176f62ad57549a0bb84773423ba8be9d089f5faba"
 dependencies = [
  "proc-macro2",
- "syn 2.0.66",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -2563,9 +3059,9 @@ dependencies = [
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.85"
+version = "1.0.86"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "22244ce15aa966053a896d1accb3a6e68469b97c7f33f284b99f0d576879fc23"
+checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77"
 dependencies = [
  "unicode-ident",
 ]
@@ -2586,7 +3082,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8021cf59c8ec9c432cfc2526ac6b8aa508ecaf29cd415f271b8406c1b851c3fd"
 dependencies = [
  "quote",
- "syn 2.0.66",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -2616,8 +3112,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4"
 dependencies = [
  "bytes",
- "heck 0.4.1",
- "itertools 0.10.5",
+ "heck 0.5.0",
+ "itertools 0.12.1",
  "log",
  "multimap",
  "once_cell",
@@ -2626,7 +3122,7 @@ dependencies = [
  "prost 0.12.6",
  "prost-types",
  "regex",
- "syn 2.0.66",
+ "syn 2.0.79",
  "tempfile",
 ]
 
@@ -2650,10 +3146,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1"
 dependencies = [
  "anyhow",
- "itertools 0.10.5",
+ "itertools 0.12.1",
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -2665,6 +3161,69 @@ dependencies = [
  "prost 0.12.6",
 ]
 
+[[package]]
+name = "pyo3"
+version = "0.22.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "15ee168e30649f7f234c3d49ef5a7a6cbf5134289bc46c29ff3155fa3221c225"
+dependencies = [
+ "cfg-if",
+ "indoc",
+ "libc",
+ "memoffset",
+ "once_cell",
+ "portable-atomic",
+ "pyo3-build-config",
+ "pyo3-ffi",
+ "pyo3-macros",
+ "unindent",
+]
+
+[[package]]
+name = "pyo3-build-config"
+version = "0.22.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e61cef80755fe9e46bb8a0b8f20752ca7676dcc07a5277d8b7768c6172e529b3"
+dependencies = [
+ "once_cell",
+ "target-lexicon",
+]
+
+[[package]]
+name = "pyo3-ffi"
+version = "0.22.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "67ce096073ec5405f5ee2b8b31f03a68e02aa10d5d4f565eca04acc41931fa1c"
+dependencies = [
+ "libc",
+ "pyo3-build-config",
+]
+
+[[package]]
+name = "pyo3-macros"
+version = "0.22.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2440c6d12bc8f3ae39f1e775266fa5122fd0c8891ce7520fa6048e683ad3de28"
+dependencies = [
+ "proc-macro2",
+ "pyo3-macros-backend",
+ "quote",
+ "syn 2.0.79",
+]
+
+[[package]]
+name = "pyo3-macros-backend"
+version = "0.22.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1be962f0e06da8f8465729ea2cb71a416d2257dff56cbe40a70d3e62a93ae5d1"
+dependencies = [
+ "heck 0.5.0",
+ "proc-macro2",
+ "pyo3-build-config",
+ "quote",
+ "syn 2.0.79",
+]
+
 [[package]]
 name = "qoi"
 version = "0.4.1"
@@ -2676,13 +3235,12 @@ dependencies = [
 
 [[package]]
 name = "quanta"
-version = "0.11.1"
+version = "0.12.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a17e662a7a8291a865152364c20c7abc5e60486ab2001e8ec10b24862de0b9ab"
+checksum = "8e5167a477619228a0b284fac2674e3c388cba90631d7b7de620e6f1fcd08da5"
 dependencies = [
  "crossbeam-utils",
  "libc",
- "mach2",
  "once_cell",
  "raw-cpuid",
  "wasi",
@@ -2698,9 +3256,9 @@ checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3"
 
 [[package]]
 name = "quote"
-version = "1.0.36"
+version = "1.0.37"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7"
+checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af"
 dependencies = [
  "proc-macro2",
 ]
@@ -2737,18 +3295,22 @@ dependencies = [
 
 [[package]]
 name = "ratatui"
-version = "0.23.0"
+version = "0.28.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2e2e4cd95294a85c3b4446e63ef054eea43e0205b1fd60120c16b74ff7ff96ad"
+checksum = "fdef7f9be5c0122f890d58bdf4d964349ba6a6161f705907526d891efabba57d"
 dependencies = [
- "bitflags 2.5.0",
+ "bitflags 2.6.0",
  "cassowary",
+ "compact_str",
  "crossterm",
- "indoc",
- "itertools 0.11.0",
+ "instability",
+ "itertools 0.13.0",
+ "lru",
  "paste",
  "strum",
+ "strum_macros",
  "unicode-segmentation",
+ "unicode-truncate",
  "unicode-width",
 ]
 
@@ -2789,26 +3351,25 @@ dependencies = [
 
 [[package]]
 name = "ravif"
-version = "0.11.5"
+version = "0.11.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bc13288f5ab39e6d7c9d501759712e6969fcc9734220846fc9ed26cae2cc4234"
+checksum = "a8f0bfd976333248de2078d350bfdf182ff96e168a24d23d2436cef320dd4bdd"
 dependencies = [
  "avif-serialize",
  "imgref",
  "loop9",
  "quick-error",
  "rav1e",
- "rayon",
  "rgb",
 ]
 
 [[package]]
 name = "raw-cpuid"
-version = "10.7.0"
+version = "11.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332"
+checksum = "cb9ee317cfe3fbd54b36a511efc1edd42e216903c9cd575e686dd68a2ba90d8d"
 dependencies = [
- "bitflags 1.3.2",
+ "bitflags 2.6.0",
 ]
 
 [[package]]
@@ -2844,18 +3405,18 @@ dependencies = [
 
 [[package]]
 name = "redox_syscall"
-version = "0.5.2"
+version = "0.5.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c82cf8cff14456045f55ec4241383baeff27af886adb72ffb2162f99911de0fd"
+checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f"
 dependencies = [
- "bitflags 2.5.0",
+ "bitflags 2.6.0",
 ]
 
 [[package]]
 name = "redox_users"
-version = "0.4.5"
+version = "0.4.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bd283d9651eeda4b2a83a43c1c91b266c40fd76ecd39a50a8c630ae69dc72891"
+checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43"
 dependencies = [
  "getrandom",
  "libredox",
@@ -2864,14 +3425,14 @@ dependencies = [
 
 [[package]]
 name = "regex"
-version = "1.10.5"
+version = "1.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f"
+checksum = "38200e5ee88914975b69f657f0801b6f6dccafd44fd9326302a4aaeecfacb1d8"
 dependencies = [
  "aho-corasick",
  "memchr",
- "regex-automata 0.4.7",
- "regex-syntax 0.8.4",
+ "regex-automata 0.4.8",
+ "regex-syntax 0.8.5",
 ]
 
 [[package]]
@@ -2885,13 +3446,13 @@ dependencies = [
 
 [[package]]
 name = "regex-automata"
-version = "0.4.7"
+version = "0.4.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df"
+checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3"
 dependencies = [
  "aho-corasick",
  "memchr",
- "regex-syntax 0.8.4",
+ "regex-syntax 0.8.5",
 ]
 
 [[package]]
@@ -2902,9 +3463,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1"
 
 [[package]]
 name = "regex-syntax"
-version = "0.8.4"
+version = "0.8.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b"
+checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"
 
 [[package]]
 name = "reqwest"
@@ -2917,10 +3478,10 @@ dependencies = [
  "encoding_rs",
  "futures-core",
  "futures-util",
- "h2",
- "http",
- "http-body",
- "hyper",
+ "h2 0.3.26",
+ "http 0.2.12",
+ "http-body 0.4.6",
+ "hyper 0.14.30",
  "hyper-tls",
  "ipnet",
  "js-sys",
@@ -2930,11 +3491,11 @@ dependencies = [
  "once_cell",
  "percent-encoding",
  "pin-project-lite",
- "rustls-pemfile",
+ "rustls-pemfile 1.0.4",
  "serde",
  "serde_json",
  "serde_urlencoded",
- "sync_wrapper",
+ "sync_wrapper 0.1.2",
  "system-configuration",
  "tokio",
  "tokio-native-tls",
@@ -2948,9 +3509,9 @@ dependencies = [
 
 [[package]]
 name = "rgb"
-version = "0.8.37"
+version = "0.8.50"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "05aaa8004b64fd573fc9d002f4e632d51ad4f026c2b5ba95fcb6c2f32c2c47d8"
+checksum = "57397d16646700483b67d2dd6511d79318f9d057fdbd21a4066aeac8b41d310a"
 dependencies = [
  "bytemuck",
 ]
@@ -2987,9 +3548,9 @@ dependencies = [
 
 [[package]]
 name = "rust-embed"
-version = "6.8.1"
+version = "8.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a36224c3276f8c4ebc8c20f158eca7ca4359c8db89991c4925132aaaf6702661"
+checksum = "fa66af4a4fdd5e7ebc276f115e895611a34739a9c1c01028383d612d550953c0"
 dependencies = [
  "rust-embed-impl",
  "rust-embed-utils",
@@ -2998,23 +3559,22 @@ dependencies = [
 
 [[package]]
 name = "rust-embed-impl"
-version = "6.8.1"
+version = "8.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "49b94b81e5b2c284684141a2fb9e2a31be90638caf040bf9afbc5a0416afe1ac"
+checksum = "6125dbc8867951125eec87294137f4e9c2c96566e61bf72c45095a7c77761478"
 dependencies = [
  "proc-macro2",
  "quote",
  "rust-embed-utils",
- "shellexpand",
- "syn 2.0.66",
+ "syn 2.0.79",
  "walkdir",
 ]
 
 [[package]]
 name = "rust-embed-utils"
-version = "7.8.1"
+version = "8.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9d38ff6bf570dc3bb7100fce9f7b60c33fa71d80e88da3f2580df4ff2bdded74"
+checksum = "2e5347777e9aacb56039b0e1f28785929a8a3b709e87482e7442c72e7c12529d"
 dependencies = [
  "sha2",
  "walkdir",
@@ -3027,21 +3587,27 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f"
 
 [[package]]
-name = "rustc_version"
-version = "0.4.0"
+name = "rustc-hash"
+version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366"
+checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
+
+[[package]]
+name = "rustc_version"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92"
 dependencies = [
  "semver",
 ]
 
 [[package]]
 name = "rustix"
-version = "0.38.34"
+version = "0.38.37"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f"
+checksum = "8acb788b847c24f28525660c4d7758620a7210875711f79e7f663cc152726811"
 dependencies = [
- "bitflags 2.5.0",
+ "bitflags 2.6.0",
  "errno",
  "libc",
  "linux-raw-sys",
@@ -3074,6 +3640,34 @@ dependencies = [
  "zeroize",
 ]
 
+[[package]]
+name = "rustls"
+version = "0.23.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f2dabaac7466917e566adb06783a81ca48944c6898a1b08b9374106dd671f4c8"
+dependencies = [
+ "aws-lc-rs",
+ "log",
+ "once_cell",
+ "rustls-pki-types",
+ "rustls-webpki",
+ "subtle",
+ "zeroize",
+]
+
+[[package]]
+name = "rustls-native-certs"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fcaf18a4f2be7326cd874a5fa579fae794320a0f388d365dca7e480e55f83f8a"
+dependencies = [
+ "openssl-probe",
+ "rustls-pemfile 2.1.3",
+ "rustls-pki-types",
+ "schannel",
+ "security-framework",
+]
+
 [[package]]
 name = "rustls-pemfile"
 version = "1.0.4"
@@ -3084,17 +3678,28 @@ dependencies = [
 ]
 
 [[package]]
-name = "rustls-pki-types"
-version = "1.7.0"
+name = "rustls-pemfile"
+version = "2.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d"
+checksum = "196fe16b00e106300d3e45ecfcb764fa292a535d7326a29a5875c579c7417425"
+dependencies = [
+ "base64 0.22.1",
+ "rustls-pki-types",
+]
+
+[[package]]
+name = "rustls-pki-types"
+version = "1.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0e696e35370c65c9c541198af4543ccd580cf17fc25d8e05c5a242b202488c55"
 
 [[package]]
 name = "rustls-webpki"
-version = "0.102.4"
+version = "0.102.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ff448f7e92e913c4b7d4c6d8e4540a1724b319b4152b8aef6d4cf8339712b33e"
+checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9"
 dependencies = [
+ "aws-lc-rs",
  "ring 0.17.8",
  "rustls-pki-types",
  "untrusted 0.9.0",
@@ -3123,11 +3728,11 @@ dependencies = [
 
 [[package]]
 name = "schannel"
-version = "0.1.23"
+version = "0.1.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534"
+checksum = "e9aaafd5a2b6e3d657ff009d82fbd630b6bd54dd4eb06f21693925cdf80f9b8b"
 dependencies = [
- "windows-sys 0.52.0",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -3136,6 +3741,12 @@ version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
 
+[[package]]
+name = "scratch"
+version = "1.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a3cf7c11c38cb994f3d40e8a8cde3bbd1f72a435e4c49e85d6553d8312306152"
+
 [[package]]
 name = "sct"
 version = "0.7.1"
@@ -3148,11 +3759,11 @@ dependencies = [
 
 [[package]]
 name = "security-framework"
-version = "2.11.0"
+version = "2.11.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c627723fd09706bacdb5cf41499e95098555af3c3c29d014dc3c458ef6be11c0"
+checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02"
 dependencies = [
- "bitflags 2.5.0",
+ "bitflags 2.6.0",
  "core-foundation",
  "core-foundation-sys",
  "libc",
@@ -3161,9 +3772,9 @@ dependencies = [
 
 [[package]]
 name = "security-framework-sys"
-version = "2.11.0"
+version = "2.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "317936bbbd05227752583946b9e66d7ce3b489f84e11a94a510b4437fef407d7"
+checksum = "ea4a292869320c0272d7bc55a5a6aafaff59b4f63404a003887b679a2e05b4b6"
 dependencies = [
  "core-foundation-sys",
  "libc",
@@ -3180,31 +3791,42 @@ dependencies = [
 
 [[package]]
 name = "serde"
-version = "1.0.203"
+version = "1.0.210"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094"
+checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
-name = "serde_derive"
-version = "1.0.203"
+name = "serde_cbor"
+version = "0.11.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba"
+checksum = "2bef2ebfde456fb76bbcf9f59315333decc4fda0b2b44b420243c11e0f5ec1f5"
+dependencies = [
+ "half 1.8.3",
+ "serde",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.210"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.79",
 ]
 
 [[package]]
 name = "serde_json"
-version = "1.0.117"
+version = "1.0.128"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "455182ea6142b14f93f4bc5320a2b31c1f266b66a4a5c858b013302a5d8cbfc3"
+checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8"
 dependencies = [
  "itoa",
+ "memchr",
  "ryu",
  "serde",
 ]
@@ -3221,9 +3843,9 @@ dependencies = [
 
 [[package]]
 name = "serde_spanned"
-version = "0.6.6"
+version = "0.6.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "79e674e01f999af37c49f70a6ede167a8a60b2503e56c5599532a65baa5969a0"
+checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1"
 dependencies = [
  "serde",
 ]
@@ -3261,13 +3883,10 @@ dependencies = [
 ]
 
 [[package]]
-name = "shellexpand"
-version = "2.1.2"
+name = "shlex"
+version = "1.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7ccc8076840c4da029af4f87e4e8daeb0fca6b87bbb02e10cb60b791450e11e4"
-dependencies = [
- "dirs 4.0.0",
-]
+checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
 
 [[package]]
 name = "signal-hook"
@@ -3281,9 +3900,9 @@ dependencies = [
 
 [[package]]
 name = "signal-hook-mio"
-version = "0.2.3"
+version = "0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "29ad2e15f37ec9a6cc544097b78a1ec90001e9f71b81338ca39f430adaca99af"
+checksum = "34db1a06d485c9142248b7a054f034b349b212551f3dfd19c94d45a754a217cd"
 dependencies = [
  "libc",
  "mio",
@@ -3382,10 +4001,10 @@ dependencies = [
 ]
 
 [[package]]
-name = "stable_deref_trait"
-version = "1.2.0"
+name = "static_assertions"
+version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"
+checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
 
 [[package]]
 name = "strsim"
@@ -3395,31 +4014,31 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
 
 [[package]]
 name = "strum"
-version = "0.25.0"
+version = "0.26.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125"
+checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06"
 dependencies = [
  "strum_macros",
 ]
 
 [[package]]
 name = "strum_macros"
-version = "0.25.3"
+version = "0.26.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "23dc1fa9ac9c169a78ba62f0b841814b7abae11bdd047b9c58f893439e309ea0"
+checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be"
 dependencies = [
- "heck 0.4.1",
+ "heck 0.5.0",
  "proc-macro2",
  "quote",
  "rustversion",
- "syn 2.0.66",
+ "syn 2.0.79",
 ]
 
 [[package]]
 name = "subtle"
-version = "2.5.0"
+version = "2.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc"
+checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
 
 [[package]]
 name = "syn"
@@ -3434,9 +4053,9 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "2.0.66"
+version = "2.0.79"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c42f3f41a2de00b01c0aaad383c5a45241efc8b2d1eda5661812fda5f3cdcff5"
+checksum = "89132cd0bf050864e1d38dc3bbc07a0eb8e7530af26344d3d2bbbef83499f590"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -3450,27 +4069,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160"
 
 [[package]]
-name = "synstructure"
-version = "0.13.1"
+name = "sync_wrapper"
+version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.66",
-]
+checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394"
 
 [[package]]
 name = "sysinfo"
-version = "0.30.12"
+version = "0.30.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "732ffa00f53e6b2af46208fba5718d9662a421049204e156328b66791ffa15ae"
+checksum = "0a5b4ddaee55fb2bea2bf0e5000747e5f5c0de765e5a5ff87f4cd106439f4bb3"
 dependencies = [
  "cfg-if",
  "core-foundation-sys",
  "libc",
  "ntapi",
  "once_cell",
+ "rayon",
  "windows",
 ]
 
@@ -3534,29 +4149,61 @@ dependencies = [
 
 [[package]]
 name = "target-lexicon"
-version = "0.12.14"
+version = "0.12.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e1fc403891a21bcfb7c37834ba66a547a8f402146eba7265b5a6d88059c9ff2f"
+checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1"
 
 [[package]]
 name = "tempfile"
-version = "3.10.1"
+version = "3.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1"
+checksum = "f0f2c9fc62d0beef6951ccffd757e241266a2c833136efbe35af6cd2567dca5b"
 dependencies = [
  "cfg-if",
  "fastrand",
+ "once_cell",
  "rustix",
- "windows-sys 0.52.0",
+ "windows-sys 0.59.0",
+]
+
+[[package]]
+name = "termcolor"
+version = "1.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755"
+dependencies = [
+ "winapi-util",
+]
+
+[[package]]
+name = "text-generation-backends-trtllm"
+version = "2.3.1-dev0"
+dependencies = [
+ "async-stream",
+ "async-trait",
+ "clap 4.5.18",
+ "cmake",
+ "cxx",
+ "cxx-build",
+ "log",
+ "parking_lot",
+ "pkg-config",
+ "text-generation-router",
+ "thiserror",
+ "tokenizers 0.19.1",
+ "tokio",
+ "tokio-stream",
+ "tracing",
+ "tracing-opentelemetry 0.24.0",
+ "tracing-subscriber",
 ]
 
 [[package]]
 name = "text-generation-benchmark"
-version = "2.0.4"
+version = "2.3.1-dev0"
 dependencies = [
  "average",
- "clap",
- "crossterm",
+ "clap 4.5.18",
  "float-ord",
  "hf-hub",
  "ratatui",
@@ -3565,7 +4212,7 @@ dependencies = [
  "tabled",
  "text-generation-client",
  "thiserror",
- "tokenizers",
+ "tokenizers 0.20.0",
  "tokio",
  "tracing",
  "tracing-subscriber",
@@ -3573,31 +4220,34 @@ dependencies = [
 
 [[package]]
 name = "text-generation-client"
-version = "2.0.4"
+version = "2.3.1-dev0"
 dependencies = [
+ "async-trait",
+ "base64 0.22.1",
  "futures",
  "grpc-metadata",
  "prost 0.12.6",
  "prost-build",
- "rand",
  "thiserror",
  "tokio",
  "tonic 0.10.2",
  "tonic-build",
- "tower",
+ "tower 0.4.13",
  "tracing",
 ]
 
 [[package]]
 name = "text-generation-launcher"
-version = "2.0.4"
+version = "2.3.1-dev0"
 dependencies = [
- "clap",
+ "clap 4.5.18",
  "ctrlc",
  "float_eq",
  "hf-hub",
- "nix",
+ "nix 0.28.0",
  "once_cell",
+ "pyo3",
+ "regex",
  "reqwest",
  "serde",
  "serde_json",
@@ -3609,64 +4259,180 @@ dependencies = [
 
 [[package]]
 name = "text-generation-router"
-version = "2.0.4"
+version = "2.3.1-dev0"
 dependencies = [
  "async-stream",
- "axum",
+ "async-trait",
+ "axum 0.7.7",
  "axum-tracing-opentelemetry",
  "base64 0.22.1",
- "clap",
+ "clap 4.5.18",
+ "csv",
  "futures",
  "futures-util",
  "hf-hub",
  "image",
  "init-tracing-opentelemetry",
+ "itertools 0.10.5",
+ "jsonschema",
+ "metrics",
+ "metrics-exporter-prometheus",
+ "minijinja",
+ "minijinja-contrib",
+ "ngrok",
+ "nohash-hasher",
+ "once_cell",
+ "opentelemetry 0.20.0",
+ "opentelemetry-otlp",
+ "pyo3",
+ "rand",
+ "regex",
+ "reqwest",
+ "serde",
+ "serde_json",
+ "sysinfo",
+ "thiserror",
+ "tokenizers 0.20.0",
+ "tokio",
+ "tokio-stream",
+ "tower-http",
+ "tracing",
+ "tracing-opentelemetry 0.21.0",
+ "tracing-subscriber",
+ "ureq",
+ "utoipa",
+ "utoipa-swagger-ui",
+ "uuid",
+ "vergen",
+]
+
+[[package]]
+name = "text-generation-router-v2"
+version = "2.3.1-dev0"
+dependencies = [
+ "async-stream",
+ "async-trait",
+ "axum 0.7.7",
+ "axum-tracing-opentelemetry",
+ "base64 0.22.1",
+ "clap 4.5.18",
+ "futures",
+ "futures-util",
+ "grpc-metadata",
+ "hf-hub",
+ "image",
+ "init-tracing-opentelemetry",
  "jsonschema",
  "metrics",
  "metrics-exporter-prometheus",
  "minijinja",
- "ngrok",
+ "minijinja-contrib",
  "nohash-hasher",
  "once_cell",
- "opentelemetry",
+ "opentelemetry 0.20.0",
  "opentelemetry-otlp",
+ "prost 0.12.6",
+ "prost-build",
  "rand",
  "regex",
  "reqwest",
  "serde",
  "serde_json",
- "text-generation-client",
+ "slotmap",
+ "text-generation-router",
  "thiserror",
- "tokenizers",
+ "tokenizers 0.20.0",
  "tokio",
  "tokio-stream",
+ "tonic 0.10.2",
+ "tonic-build",
+ "tower 0.4.13",
  "tower-http",
  "tracing",
- "tracing-opentelemetry",
+ "tracing-opentelemetry 0.21.0",
  "tracing-subscriber",
  "utoipa",
  "utoipa-swagger-ui",
- "vergen",
+]
+
+[[package]]
+name = "text-generation-router-v3"
+version = "2.3.1-dev0"
+dependencies = [
+ "async-stream",
+ "async-trait",
+ "axum 0.7.7",
+ "axum-tracing-opentelemetry",
+ "base64 0.22.1",
+ "clap 4.5.18",
+ "criterion",
+ "futures",
+ "futures-util",
+ "grpc-metadata",
+ "hf-hub",
+ "image",
+ "init-tracing-opentelemetry",
+ "itertools 0.13.0",
+ "jsonschema",
+ "metrics",
+ "metrics-exporter-prometheus",
+ "minijinja",
+ "minijinja-contrib",
+ "nohash-hasher",
+ "once_cell",
+ "opentelemetry 0.20.0",
+ "opentelemetry-otlp",
+ "prost 0.12.6",
+ "prost-build",
+ "rand",
+ "regex",
+ "reqwest",
+ "serde",
+ "serde_json",
+ "slotmap",
+ "text-generation-router",
+ "thiserror",
+ "tokenizers 0.20.0",
+ "tokio",
+ "tokio-stream",
+ "tonic 0.10.2",
+ "tonic-build",
+ "tower 0.4.13",
+ "tower-http",
+ "tracing",
+ "tracing-opentelemetry 0.21.0",
+ "tracing-subscriber",
+ "utoipa",
+ "utoipa-swagger-ui",
+]
+
+[[package]]
+name = "textwrap"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060"
+dependencies = [
+ "unicode-width",
 ]
 
 [[package]]
 name = "thiserror"
-version = "1.0.61"
+version = "1.0.64"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c546c80d6be4bc6a00c0f01730c08df82eaa7a7a61f11d656526506112cc1709"
+checksum = "d50af8abc119fb8bb6dbabcfa89656f46f84aa0ac7688088608076ad2b459a84"
 dependencies = [
  "thiserror-impl",
 ]
 
 [[package]]
 name = "thiserror-impl"
-version = "1.0.61"
+version = "1.0.64"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533"
+checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -3724,15 +4490,30 @@ dependencies = [
 ]
 
 [[package]]
-name = "tinystr"
-version = "0.7.6"
+name = "tinytemplate"
+version = "1.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f"
+checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc"
 dependencies = [
- "displaydoc",
- "zerovec",
+ "serde",
+ "serde_json",
 ]
 
+[[package]]
+name = "tinyvec"
+version = "1.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938"
+dependencies = [
+ "tinyvec_macros",
+]
+
+[[package]]
+name = "tinyvec_macros"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
+
 [[package]]
 name = "tokenizers"
 version = "0.19.1"
@@ -3756,7 +4537,40 @@ dependencies = [
  "rayon",
  "rayon-cond",
  "regex",
- "regex-syntax 0.8.4",
+ "regex-syntax 0.8.5",
+ "serde",
+ "serde_json",
+ "spm_precompiled",
+ "thiserror",
+ "unicode-normalization-alignments",
+ "unicode-segmentation",
+ "unicode_categories",
+]
+
+[[package]]
+name = "tokenizers"
+version = "0.20.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c8a24d7f7d6be5b9d1377418b893ab1808af0074f5d1bb2c64784452ddd2aa70"
+dependencies = [
+ "aho-corasick",
+ "derive_builder",
+ "esaxx-rs",
+ "getrandom",
+ "hf-hub",
+ "indicatif",
+ "itertools 0.12.1",
+ "lazy_static",
+ "log",
+ "macro_rules_attribute",
+ "monostate",
+ "onig",
+ "paste",
+ "rand",
+ "rayon",
+ "rayon-cond",
+ "regex",
+ "regex-syntax 0.8.5",
  "serde",
  "serde_json",
  "spm_precompiled",
@@ -3768,21 +4582,20 @@ dependencies = [
 
 [[package]]
 name = "tokio"
-version = "1.38.0"
+version = "1.40.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ba4f4a02a7a80d6f274636f0aa95c7e383b912d41fe721a31f29e29698585a4a"
+checksum = "e2b070231665d27ad9ec9b8df639893f46727666c6767db40317fbe920a5d998"
 dependencies = [
  "backtrace",
  "bytes",
  "libc",
  "mio",
- "num_cpus",
  "parking_lot",
  "pin-project-lite",
  "signal-hook-registry",
  "socket2",
  "tokio-macros",
- "windows-sys 0.48.0",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]
@@ -3797,13 +4610,13 @@ dependencies = [
 
 [[package]]
 name = "tokio-macros"
-version = "2.3.0"
+version = "2.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a"
+checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -3828,10 +4641,21 @@ dependencies = [
 ]
 
 [[package]]
-name = "tokio-stream"
-version = "0.1.15"
+name = "tokio-rustls"
+version = "0.26.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "267ac89e0bec6e691e5813911606935d77c476ff49024f98abcea3e7b15e37af"
+checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4"
+dependencies = [
+ "rustls 0.23.13",
+ "rustls-pki-types",
+ "tokio",
+]
+
+[[package]]
+name = "tokio-stream"
+version = "0.1.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4f4e6ce100d0eb49a2734f8c0812bcd324cf357d21810932c5df6b96ef2b86f1"
 dependencies = [
  "futures-core",
  "pin-project-lite",
@@ -3840,9 +4664,9 @@ dependencies = [
 
 [[package]]
 name = "tokio-util"
-version = "0.7.11"
+version = "0.7.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1"
+checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a"
 dependencies = [
  "bytes",
  "futures-core",
@@ -3854,9 +4678,9 @@ dependencies = [
 
 [[package]]
 name = "toml"
-version = "0.8.14"
+version = "0.8.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6f49eb2ab21d2f26bd6db7bf383edc527a7ebaee412d17af4d40fdccd442f335"
+checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e"
 dependencies = [
  "serde",
  "serde_spanned",
@@ -3866,20 +4690,20 @@ dependencies = [
 
 [[package]]
 name = "toml_datetime"
-version = "0.6.6"
+version = "0.6.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4badfd56924ae69bcc9039335b2e017639ce3f9b001c393c1b2d1ef846ce2cbf"
+checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41"
 dependencies = [
  "serde",
 ]
 
 [[package]]
 name = "toml_edit"
-version = "0.22.14"
+version = "0.22.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f21c7aaf97f1bd9ca9d4f9e73b0a6c74bd5afef56f2bc931943a6e1c37e04e38"
+checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5"
 dependencies = [
- "indexmap 2.2.6",
+ "indexmap 2.5.0",
  "serde",
  "serde_spanned",
  "toml_datetime",
@@ -3893,22 +4717,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3082666a3a6433f7f511c7192923fa1fe07c69332d3c6a2e6bb040b569199d5a"
 dependencies = [
  "async-trait",
- "axum",
+ "axum 0.6.20",
  "base64 0.21.7",
  "bytes",
  "futures-core",
  "futures-util",
- "h2",
- "http",
- "http-body",
- "hyper",
+ "h2 0.3.26",
+ "http 0.2.12",
+ "http-body 0.4.6",
+ "hyper 0.14.30",
  "hyper-timeout",
  "percent-encoding",
  "pin-project",
  "prost 0.11.9",
  "tokio",
  "tokio-stream",
- "tower",
+ "tower 0.4.13",
  "tower-layer",
  "tower-service",
  "tracing",
@@ -3922,20 +4746,20 @@ checksum = "d560933a0de61cf715926b9cac824d4c883c2c43142f787595e48280c40a1d0e"
 dependencies = [
  "async-stream",
  "async-trait",
- "axum",
+ "axum 0.6.20",
  "base64 0.21.7",
  "bytes",
- "h2",
- "http",
- "http-body",
- "hyper",
+ "h2 0.3.26",
+ "http 0.2.12",
+ "http-body 0.4.6",
+ "hyper 0.14.30",
  "hyper-timeout",
  "percent-encoding",
  "pin-project",
  "prost 0.12.6",
  "tokio",
  "tokio-stream",
- "tower",
+ "tower 0.4.13",
  "tower-layer",
  "tower-service",
  "tracing",
@@ -3951,7 +4775,7 @@ dependencies = [
  "proc-macro2",
  "prost-build",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -3975,18 +4799,32 @@ dependencies = [
 ]
 
 [[package]]
-name = "tower-http"
-version = "0.4.4"
+name = "tower"
+version = "0.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "61c5bb1d698276a2443e5ecfabc1008bf15a36c12e6a7176e7bf089ea9131140"
+checksum = "2873938d487c3cfb9aed7546dc9f2711d867c9f90c46b889989a2cb84eba6b4f"
 dependencies = [
- "bitflags 2.5.0",
- "bytes",
  "futures-core",
  "futures-util",
- "http",
- "http-body",
- "http-range-header",
+ "pin-project-lite",
+ "sync_wrapper 0.1.2",
+ "tokio",
+ "tower-layer",
+ "tower-service",
+ "tracing",
+]
+
+[[package]]
+name = "tower-http"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5"
+dependencies = [
+ "bitflags 2.6.0",
+ "bytes",
+ "http 1.1.0",
+ "http-body 1.0.1",
+ "http-body-util",
  "pin-project-lite",
  "tower-layer",
  "tower-service",
@@ -3994,15 +4832,15 @@ dependencies = [
 
 [[package]]
 name = "tower-layer"
-version = "0.3.2"
+version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0"
+checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e"
 
 [[package]]
 name = "tower-service"
-version = "0.3.2"
+version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52"
+checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3"
 
 [[package]]
 name = "tracing"
@@ -4024,7 +4862,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -4066,8 +4904,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "75327c6b667828ddc28f5e3f169036cb793c3f588d83bf0f262a7f062ffed3c8"
 dependencies = [
  "once_cell",
- "opentelemetry",
- "opentelemetry_sdk",
+ "opentelemetry 0.20.0",
+ "opentelemetry_sdk 0.20.0",
  "smallvec",
  "tracing",
  "tracing-core",
@@ -4076,16 +4914,51 @@ dependencies = [
 ]
 
 [[package]]
-name = "tracing-opentelemetry-instrumentation-sdk"
-version = "0.14.2"
+name = "tracing-opentelemetry"
+version = "0.22.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f523eba1b52bb854b804d43a039aafeaee5a623015065adbfef8016825319c15"
+checksum = "c67ac25c5407e7b961fafc6f7e9aa5958fd297aada2d20fa2ae1737357e55596"
 dependencies = [
- "http",
- "opentelemetry-http",
- "opentelemetry_api",
+ "js-sys",
+ "once_cell",
+ "opentelemetry 0.21.0",
+ "opentelemetry_sdk 0.21.2",
+ "smallvec",
  "tracing",
- "tracing-opentelemetry",
+ "tracing-core",
+ "tracing-log 0.2.0",
+ "tracing-subscriber",
+ "web-time 0.2.4",
+]
+
+[[package]]
+name = "tracing-opentelemetry"
+version = "0.24.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f68803492bf28ab40aeccaecc7021096bd256baf7ca77c3d425d89b35a7be4e4"
+dependencies = [
+ "js-sys",
+ "once_cell",
+ "opentelemetry 0.23.0",
+ "opentelemetry_sdk 0.23.0",
+ "smallvec",
+ "tracing",
+ "tracing-core",
+ "tracing-log 0.2.0",
+ "tracing-subscriber",
+ "web-time 1.1.0",
+]
+
+[[package]]
+name = "tracing-opentelemetry-instrumentation-sdk"
+version = "0.16.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9920abb6a3ee3a2af7d30c9ff02900f8481935d36723c3da95cf807468218e8c"
+dependencies = [
+ "http 1.1.0",
+ "opentelemetry 0.21.0",
+ "tracing",
+ "tracing-opentelemetry 0.22.0",
 ]
 
 [[package]]
@@ -4141,10 +5014,25 @@ dependencies = [
 ]
 
 [[package]]
-name = "unicode-ident"
-version = "1.0.12"
+name = "unicode-bidi"
+version = "0.3.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
+checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75"
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe"
+
+[[package]]
+name = "unicode-normalization"
+version = "0.1.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956"
+dependencies = [
+ "tinyvec",
+]
 
 [[package]]
 name = "unicode-normalization-alignments"
@@ -4157,15 +5045,26 @@ dependencies = [
 
 [[package]]
 name = "unicode-segmentation"
-version = "1.11.0"
+version = "1.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202"
+checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493"
+
+[[package]]
+name = "unicode-truncate"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b3644627a5af5fa321c95b9b235a72fd24cd29c648c2c379431e6628655627bf"
+dependencies = [
+ "itertools 0.13.0",
+ "unicode-segmentation",
+ "unicode-width",
+]
 
 [[package]]
 name = "unicode-width"
-version = "0.1.13"
+version = "0.1.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0336d538f7abc86d282a4189614dfaa90810dfc2c6f6427eaf88e16311dd225d"
+checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af"
 
 [[package]]
 name = "unicode_categories"
@@ -4173,6 +5072,12 @@ version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e"
 
+[[package]]
+name = "unindent"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce"
+
 [[package]]
 name = "untrusted"
 version = "0.7.1"
@@ -4207,9 +5112,9 @@ dependencies = [
 
 [[package]]
 name = "url"
-version = "2.5.1"
+version = "2.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f7c25da092f0a868cdf09e8674cd3b7ef3a7d92a24253e663a2fb85e2496de56"
+checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c"
 dependencies = [
  "form_urlencoded",
  "idna",
@@ -4222,18 +5127,6 @@ version = "2.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da"
 
-[[package]]
-name = "utf16_iter"
-version = "1.0.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246"
-
-[[package]]
-name = "utf8_iter"
-version = "1.0.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
-
 [[package]]
 name = "utf8parse"
 version = "0.2.2"
@@ -4242,11 +5135,11 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
 
 [[package]]
 name = "utoipa"
-version = "3.5.0"
+version = "4.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d82b1bc5417102a73e8464c686eef947bdfb99fcdfc0a4f228e81afa9526470a"
+checksum = "c5afb1a60e207dca502682537fefcfd9921e71d0b83e9576060f09abc6efab23"
 dependencies = [
- "indexmap 2.2.6",
+ "indexmap 2.5.0",
  "serde",
  "serde_json",
  "utoipa-gen",
@@ -4254,24 +5147,24 @@ dependencies = [
 
 [[package]]
 name = "utoipa-gen"
-version = "3.5.0"
+version = "4.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "05d96dcd6fc96f3df9b3280ef480770af1b7c5d14bc55192baa9b067976d920c"
+checksum = "7bf0e16c02bc4bf5322ab65f10ab1149bdbcaa782cba66dc7057370a3f8190be"
 dependencies = [
  "proc-macro-error",
  "proc-macro2",
  "quote",
  "regex",
- "syn 2.0.66",
+ "syn 2.0.79",
 ]
 
 [[package]]
 name = "utoipa-swagger-ui"
-version = "3.1.5"
+version = "6.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "84614caa239fb25b2bb373a52859ffd94605ceb256eeb1d63436325cf81e3653"
+checksum = "0b39868d43c011961e04b41623e050aedf2cc93652562ff7935ce0f819aaf2da"
 dependencies = [
- "axum",
+ "axum 0.7.7",
  "mime_guess",
  "regex",
  "rust-embed",
@@ -4283,9 +5176,25 @@ dependencies = [
 
 [[package]]
 name = "uuid"
-version = "1.8.0"
+version = "1.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a183cf7feeba97b4dd1c0d46788634f6221d87fa961b305bed08c851829efcc0"
+checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314"
+dependencies = [
+ "getrandom",
+ "rand",
+ "uuid-macro-internal",
+]
+
+[[package]]
+name = "uuid-macro-internal"
+version = "1.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ee1cd046f83ea2c4e920d6ee9f7c3537ef928d75dce5d84a87c2c5d6b3999a3a"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.79",
+]
 
 [[package]]
 name = "v_frame"
@@ -4312,9 +5221,9 @@ checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
 
 [[package]]
 name = "vergen"
-version = "8.3.1"
+version = "8.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e27d6bdd219887a9eadd19e1c34f32e47fa332301184935c6d9bca26f3cca525"
+checksum = "2990d9ea5967266ea0ccf413a4aa5c42a93dbcfda9cb49a97de6931726b12566"
 dependencies = [
  "anyhow",
  "cargo_metadata",
@@ -4334,9 +5243,9 @@ checksum = "852e951cb7832cb45cb1169900d19760cfa39b82bc0ea9c0e5a14ae88411c98b"
 
 [[package]]
 name = "version_check"
-version = "0.9.4"
+version = "0.9.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
+checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
 
 [[package]]
 name = "walkdir"
@@ -4365,34 +5274,35 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
 
 [[package]]
 name = "wasm-bindgen"
-version = "0.2.92"
+version = "0.2.93"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8"
+checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5"
 dependencies = [
  "cfg-if",
+ "once_cell",
  "wasm-bindgen-macro",
 ]
 
 [[package]]
 name = "wasm-bindgen-backend"
-version = "0.2.92"
+version = "0.2.93"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da"
+checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b"
 dependencies = [
  "bumpalo",
  "log",
  "once_cell",
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.79",
  "wasm-bindgen-shared",
 ]
 
 [[package]]
 name = "wasm-bindgen-futures"
-version = "0.4.42"
+version = "0.4.43"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0"
+checksum = "61e9300f63a621e96ed275155c108eb6f843b6a26d053f122ab69724559dc8ed"
 dependencies = [
  "cfg-if",
  "js-sys",
@@ -4402,9 +5312,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro"
-version = "0.2.92"
+version = "0.2.93"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726"
+checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf"
 dependencies = [
  "quote",
  "wasm-bindgen-macro-support",
@@ -4412,28 +5322,48 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro-support"
-version = "0.2.92"
+version = "0.2.93"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7"
+checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.66",
+ "syn 2.0.79",
  "wasm-bindgen-backend",
  "wasm-bindgen-shared",
 ]
 
 [[package]]
 name = "wasm-bindgen-shared"
-version = "0.2.92"
+version = "0.2.93"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96"
+checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484"
 
 [[package]]
 name = "web-sys"
-version = "0.3.69"
+version = "0.3.70"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef"
+checksum = "26fdeaafd9bd129f65e7c031593c24d62186301e0c72c8978fa1678be7d532c0"
+dependencies = [
+ "js-sys",
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "web-time"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aa30049b1c872b72c89866d458eae9f20380ab280ffd1b1e18df2d3e2d98cfe0"
+dependencies = [
+ "js-sys",
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "web-time"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb"
 dependencies = [
  "js-sys",
  "wasm-bindgen",
@@ -4451,9 +5381,9 @@ dependencies = [
 
 [[package]]
 name = "webpki-roots"
-version = "0.26.2"
+version = "0.26.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3c452ad30530b54a4d8e71952716a212b08efd0f3562baa66c29a618b07da7c3"
+checksum = "841c67bff177718f1d4dfefde8d8f0e78f9b6589319ba88312f567fc5841a958"
 dependencies = [
  "rustls-pki-types",
 ]
@@ -4464,6 +5394,18 @@ version = "0.1.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "53a85b86a771b1c87058196170769dd264f66c0782acf1ae6cc51bfd64b39082"
 
+[[package]]
+name = "which"
+version = "4.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7"
+dependencies = [
+ "either",
+ "home",
+ "once_cell",
+ "rustix",
+]
+
 [[package]]
 name = "winapi"
 version = "0.3.9"
@@ -4482,11 +5424,11 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
 
 [[package]]
 name = "winapi-util"
-version = "0.1.8"
+version = "0.1.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b"
+checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb"
 dependencies = [
- "windows-sys 0.52.0",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -4502,7 +5444,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be"
 dependencies = [
  "windows-core",
- "windows-targets 0.52.5",
+ "windows-targets 0.52.6",
 ]
 
 [[package]]
@@ -4511,7 +5453,7 @@ version = "0.52.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9"
 dependencies = [
- "windows-targets 0.52.5",
+ "windows-targets 0.52.6",
 ]
 
 [[package]]
@@ -4538,7 +5480,16 @@ version = "0.52.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
 dependencies = [
- "windows-targets 0.52.5",
+ "windows-targets 0.52.6",
+]
+
+[[package]]
+name = "windows-sys"
+version = "0.59.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
+dependencies = [
+ "windows-targets 0.52.6",
 ]
 
 [[package]]
@@ -4573,18 +5524,18 @@ dependencies = [
 
 [[package]]
 name = "windows-targets"
-version = "0.52.5"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb"
+checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
 dependencies = [
- "windows_aarch64_gnullvm 0.52.5",
- "windows_aarch64_msvc 0.52.5",
- "windows_i686_gnu 0.52.5",
+ "windows_aarch64_gnullvm 0.52.6",
+ "windows_aarch64_msvc 0.52.6",
+ "windows_i686_gnu 0.52.6",
  "windows_i686_gnullvm",
- "windows_i686_msvc 0.52.5",
- "windows_x86_64_gnu 0.52.5",
- "windows_x86_64_gnullvm 0.52.5",
- "windows_x86_64_msvc 0.52.5",
+ "windows_i686_msvc 0.52.6",
+ "windows_x86_64_gnu 0.52.6",
+ "windows_x86_64_gnullvm 0.52.6",
+ "windows_x86_64_msvc 0.52.6",
 ]
 
 [[package]]
@@ -4601,9 +5552,9 @@ checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8"
 
 [[package]]
 name = "windows_aarch64_gnullvm"
-version = "0.52.5"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263"
+checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
 
 [[package]]
 name = "windows_aarch64_msvc"
@@ -4619,9 +5570,9 @@ checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc"
 
 [[package]]
 name = "windows_aarch64_msvc"
-version = "0.52.5"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6"
+checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
 
 [[package]]
 name = "windows_i686_gnu"
@@ -4637,15 +5588,15 @@ checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e"
 
 [[package]]
 name = "windows_i686_gnu"
-version = "0.52.5"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670"
+checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
 
 [[package]]
 name = "windows_i686_gnullvm"
-version = "0.52.5"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9"
+checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
 
 [[package]]
 name = "windows_i686_msvc"
@@ -4661,9 +5612,9 @@ checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406"
 
 [[package]]
 name = "windows_i686_msvc"
-version = "0.52.5"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf"
+checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
 
 [[package]]
 name = "windows_x86_64_gnu"
@@ -4679,9 +5630,9 @@ checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e"
 
 [[package]]
 name = "windows_x86_64_gnu"
-version = "0.52.5"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9"
+checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
 
 [[package]]
 name = "windows_x86_64_gnullvm"
@@ -4697,9 +5648,9 @@ checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc"
 
 [[package]]
 name = "windows_x86_64_gnullvm"
-version = "0.52.5"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596"
+checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
 
 [[package]]
 name = "windows_x86_64_msvc"
@@ -4715,15 +5666,15 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"
 
 [[package]]
 name = "windows_x86_64_msvc"
-version = "0.52.5"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0"
+checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
 
 [[package]]
 name = "winnow"
-version = "0.6.13"
+version = "0.6.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "59b5e5f6c299a3c7890b876a2a587f3115162487e704907d9b6cd29473052ba1"
+checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b"
 dependencies = [
  "memchr",
 ]
@@ -4738,81 +5689,25 @@ dependencies = [
  "windows-sys 0.48.0",
 ]
 
-[[package]]
-name = "write16"
-version = "1.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936"
-
-[[package]]
-name = "writeable"
-version = "0.5.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51"
-
-[[package]]
-name = "yoke"
-version = "0.7.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6c5b1314b079b0930c31e3af543d8ee1757b1951ae1e1565ec704403a7240ca5"
-dependencies = [
- "serde",
- "stable_deref_trait",
- "yoke-derive",
- "zerofrom",
-]
-
-[[package]]
-name = "yoke-derive"
-version = "0.7.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.66",
- "synstructure",
-]
-
 [[package]]
 name = "zerocopy"
-version = "0.7.34"
+version = "0.7.35"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ae87e3fcd617500e5d106f0380cf7b77f3c6092aae37191433159dda23cfb087"
+checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0"
 dependencies = [
+ "byteorder",
  "zerocopy-derive",
 ]
 
 [[package]]
 name = "zerocopy-derive"
-version = "0.7.34"
+version = "0.7.35"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "15e934569e47891f7d9411f1a451d947a60e000ab3bd24fbb970f000387d1b3b"
+checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.66",
-]
-
-[[package]]
-name = "zerofrom"
-version = "0.1.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "91ec111ce797d0e0784a1116d0ddcdbea84322cd79e5d5ad173daeba4f93ab55"
-dependencies = [
- "zerofrom-derive",
-]
-
-[[package]]
-name = "zerofrom-derive"
-version = "0.1.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.66",
- "synstructure",
+ "syn 2.0.79",
 ]
 
 [[package]]
@@ -4821,28 +5716,6 @@ version = "1.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde"
 
-[[package]]
-name = "zerovec"
-version = "0.10.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bb2cc8827d6c0994478a15c53f374f46fbd41bea663d809b14744bc42e6b109c"
-dependencies = [
- "yoke",
- "zerofrom",
- "zerovec-derive",
-]
-
-[[package]]
-name = "zerovec-derive"
-version = "0.10.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97cf56601ee5052b4417d90c8755c6683473c926039908196cf35d99f893ebe7"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.66",
-]
-
 [[package]]
 name = "zip"
 version = "0.6.6"
@@ -4872,9 +5745,9 @@ dependencies = [
 
 [[package]]
 name = "zune-jpeg"
-version = "0.4.11"
+version = "0.4.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ec866b44a2a1fd6133d363f073ca1b179f438f99e7e5bfb1e33f7181facfe448"
+checksum = "16099418600b4d8f028622f73ff6e3deaabdff330fb9a2a131dea781ee8b0768"
 dependencies = [
  "zune-core",
 ]
diff --git a/Cargo.toml b/Cargo.toml
index 83972519..a51c6d04 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,27 +1,53 @@
 [workspace]
 members = [
-    "benchmark",
-    "router",
-    "router/client",
-    "router/grpc-metadata",
-    "launcher"
+  "benchmark",
+  "backends/v2",
+  "backends/v3",
+  "backends/grpc-metadata",
+  "backends/trtllm",
+  "launcher",
+  "router"
+]
+default-members = [
+  "benchmark",
+  "backends/v2",
+  "backends/v3",
+  "backends/grpc-metadata",
+  # "backends/trtllm",
+  "launcher",
+  "router"
 ]
 resolver = "2"
 
 [workspace.package]
-version = "2.0.4"
+version = "2.3.1"
 edition = "2021"
 authors = ["Olivier Dehaene"]
 homepage = "https://github.com/huggingface/text-generation-inference"
 
 [workspace.dependencies]
+base64 = "0.22.0"
 tokenizers = { version = "0.20.0", features = ["http"] }
 hf-hub = { version = "0.3.1", features = ["tokio"] }
+metrics = { version = "0.23.0" }
+metrics-exporter-prometheus = { version = "0.15.1", features = [] }
+minijinja = { version = "2.2.0", features = ["json"] }
+minijinja-contrib = { version = "2.0.2", features = ["pycompat"] }
+pyo3 = { version = "0.22.2", features = ["auto-initialize"] }
 
 [profile.release]
+incremental = true
+
+[profile.release-binary]
+inherits = "release"
 debug = 1
 incremental = true
+panic = "abort"
+
+[profile.release-opt]
+inherits = "release"
+debug = 0
+incremental = false
 lto = "fat"
 opt-level = 3
 codegen-units = 1
-panic = "abort"
diff --git a/Dockerfile b/Dockerfile
index d8198a3a..9276377f 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,18 +1,24 @@
 # Rust builder
-FROM lukemathwalker/cargo-chef:latest-rust-1.78 AS chef
+FROM lukemathwalker/cargo-chef:latest-rust-1.80 AS chef
 WORKDIR /usr/src
 
-FROM chef as planner
+ARG CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse
+
+FROM chef AS planner
+COPY Cargo.lock Cargo.lock
 COPY Cargo.toml Cargo.toml
 COPY rust-toolchain.toml rust-toolchain.toml
 COPY proto proto
 COPY benchmark benchmark
 COPY router router
+COPY backends backends
 COPY launcher launcher
 RUN cargo chef prepare --recipe-path recipe.json
 
 FROM chef AS builder
 
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+    python3.11-dev
 RUN PROTOC_ZIP=protoc-21.12-linux-x86_64.zip && \
     curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v21.12/$PROTOC_ZIP && \
     unzip -o $PROTOC_ZIP -d /usr/local bin/protoc && \
@@ -20,22 +26,29 @@ RUN PROTOC_ZIP=protoc-21.12-linux-x86_64.zip && \
     rm -f $PROTOC_ZIP
 
 COPY --from=planner /usr/src/recipe.json recipe.json
-COPY Cargo.lock Cargo.lock
-RUN cargo chef cook --release --recipe-path recipe.json
+RUN cargo chef cook --profile release-opt --recipe-path recipe.json
+
+ARG GIT_SHA
+ARG DOCKER_LABEL
 
 COPY Cargo.toml Cargo.toml
 COPY rust-toolchain.toml rust-toolchain.toml
 COPY proto proto
 COPY benchmark benchmark
 COPY router router
+COPY backends backends
 COPY launcher launcher
-RUN cargo build --release
+RUN cargo build --profile release-opt
 
 # Text Generation Inference base image
 FROM vault.habana.ai/gaudi-docker/1.18.0/ubuntu22.04/habanalabs/pytorch-installer-2.4.0:latest as base
 
+ENV ATTENTION=default
+ENV PREFIX_CACHING=0
+ENV PREFILL_CHUNKING=0
+
 # Text Generation Inference base env
-ENV HUGGINGFACE_HUB_CACHE=/data \
+ENV HF_HOME=/data \
     HF_HUB_ENABLE_HF_TRANSFER=1 \
     PORT=80
 
@@ -51,6 +64,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins
         make \
         curl \
         git \
+        python3.11-dev \
         && rm -rf /var/lib/apt/lists/*
 
 # Install server
@@ -59,21 +73,33 @@ COPY server server
 COPY server/Makefile server/Makefile
 RUN cd server && \
     make gen-server && \
-    pip install -r requirements.txt && \
+    pip install --no-deps -r requirements.txt && \
     bash ./dill-0.3.8-patch.sh && \
     pip install git+https://github.com/HabanaAI/DeepSpeed.git@1.18.0 && \
     BUILD_CUDA_EXT=0 pip install git+https://github.com/AutoGPTQ/AutoGPTQ.git@097dd04e --no-build-isolation && \
     pip install . --no-cache-dir
 
 # Install benchmarker
-COPY --from=builder /usr/src/target/release/text-generation-benchmark /usr/local/bin/text-generation-benchmark
+COPY --from=builder /usr/src/target/release-opt/text-generation-benchmark /usr/local/bin/text-generation-benchmark
 # Install router
-COPY --from=builder /usr/src/target/release/text-generation-router /usr/local/bin/text-generation-router
+COPY --from=builder /usr/src/target/release-opt/text-generation-router /usr/local/bin/text-generation-router
 # Install launcher
-COPY --from=builder /usr/src/target/release/text-generation-launcher /usr/local/bin/text-generation-launcher
+COPY --from=builder /usr/src/target/release-opt/text-generation-launcher /usr/local/bin/text-generation-launcher
+
+
+# AWS Sagemaker compatible image
+FROM base AS sagemaker
+
+COPY sagemaker-entrypoint.sh entrypoint.sh
+RUN chmod +x entrypoint.sh
+
+ENTRYPOINT ["./entrypoint.sh"]
 
 # Final image
 FROM base
 
-ENTRYPOINT ["text-generation-launcher"]
+COPY ./tgi-entrypoint.sh /tgi-entrypoint.sh
+RUN chmod +x /tgi-entrypoint.sh
+
+ENTRYPOINT ["/tgi-entrypoint.sh"]
 CMD ["--json-output"]
diff --git a/Dockerfile.nix b/Dockerfile.nix
new file mode 100644
index 00000000..f1e7e0f5
--- /dev/null
+++ b/Dockerfile.nix
@@ -0,0 +1,24 @@
+# Build the image and get out the docker file:
+#
+# docker build -t tgi-nix-builder -f Dockerfile.nix
+# docker run --log-driver=none tgi-nix-builder | docker load
+
+FROM nixos/nix:2.18.8 AS builder
+RUN echo "experimental-features = nix-command flakes" >> /etc/nix/nix.conf
+RUN nix profile install nixpkgs#cachix
+RUN cachix use text-generation-inference
+WORKDIR /root
+ADD . .
+RUN nix build .
+RUN mkdir /tmp/nix-store-closure
+RUN cp -R $(nix-store -qR result/) /tmp/nix-store-closure
+
+FROM ubuntu:24.04
+
+WORKDIR /app
+
+# Copy /nix/store
+COPY --from=builder /tmp/nix-store-closure /nix/store
+COPY --from=builder /root/result /app
+RUN ldconfig
+CMD ["ldconfig", "/app/bin/text-generation-launcher"]
diff --git a/Dockerfile.trtllm b/Dockerfile.trtllm
new file mode 100644
index 00000000..4543ae80
--- /dev/null
+++ b/Dockerfile.trtllm
@@ -0,0 +1,23 @@
+# All the tooling for CUDA
+FROM nvidia/cuda:12.4.1-cudnn-devel-ubuntu22.04 AS cuda-builder
+
+WORKDIR /usr/src/tgi/backends/trtllm
+RUN apt update && apt install -y cmake git git-lfs gcc g++ ninja-build libopenmpi-dev python3-dev python3-pip wget
+
+COPY . /usr/src/tgi
+RUN chmod +x scripts/install_tensorrt.sh && scripts/install_tensorrt.sh
+RUN cmake -G Ninja -B build -DTRT_LIB_DIR=/usr/local/tensorrt/lib -DTRT_INCLUDE_DIR=/usr/local/tensorrt/include .
+RUN cmake --build build --parallel -t tgi_trtllm_backend_impl
+
+# All the tooling for Rust
+FROM lukemathwalker/cargo-chef:latest-rust-1.79 AS chef
+WORKDIR /usr/src
+
+# Include CUDA related libraries and tools to the Rust based image
+COPY --from=cuda-builder /usr/local/cuda /usr/local/cuda
+COPY --from=cuda-builder /usr/local/tensorrt /usr/local/tensorrt
+COPY --from=cuda-builder /usr/src/tgi/backends/trtllm/build /usr/local/tgi/trtllm/build
+ENV PATH=/usr/local/cuda/bin:$PATH
+ENV LD_LIBRARY_PATH=/usr/local/tensorrt/lib:$LD_LIBRARY_PATH
+
+RUN apt update && apt install -y cmake git gcc g++ ninja-build libopenmpi3
diff --git a/Dockerfile_amd b/Dockerfile_amd
index 92dd0ea8..0b059f8c 100644
--- a/Dockerfile_amd
+++ b/Dockerfile_amd
@@ -1,23 +1,24 @@
 # Rust builder
-FROM lukemathwalker/cargo-chef:latest-rust-1.78 AS chef
+FROM lukemathwalker/cargo-chef:latest-rust-1.80 AS chef
 WORKDIR /usr/src
 
 ARG CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse
 
-FROM chef as planner
+FROM chef AS planner
+COPY Cargo.lock Cargo.lock
 COPY Cargo.toml Cargo.toml
 COPY rust-toolchain.toml rust-toolchain.toml
 COPY proto proto
 COPY benchmark benchmark
 COPY router router
+COPY backends backends
 COPY launcher launcher
 RUN cargo chef prepare --recipe-path recipe.json
 
 FROM chef AS builder
 
-ARG GIT_SHA
-ARG DOCKER_LABEL
-
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+    python3.11-dev
 RUN PROTOC_ZIP=protoc-21.12-linux-x86_64.zip && \
     curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v21.12/$PROTOC_ZIP && \
     unzip -o $PROTOC_ZIP -d /usr/local bin/protoc && \
@@ -25,18 +26,22 @@ RUN PROTOC_ZIP=protoc-21.12-linux-x86_64.zip && \
     rm -f $PROTOC_ZIP
 
 COPY --from=planner /usr/src/recipe.json recipe.json
-RUN cargo chef cook --release --recipe-path recipe.json
+RUN cargo chef cook --profile release-opt --recipe-path recipe.json
+
+ARG GIT_SHA
+ARG DOCKER_LABEL
 
 COPY Cargo.toml Cargo.toml
 COPY rust-toolchain.toml rust-toolchain.toml
 COPY proto proto
 COPY benchmark benchmark
 COPY router router
+COPY backends backends
 COPY launcher launcher
-RUN cargo build --release
+RUN cargo build --profile release-opt
 
 # Text Generation Inference base image for RoCm
-FROM rocm/dev-ubuntu-22.04:6.1.1_hip_update as base
+FROM rocm/dev-ubuntu-22.04:6.2 AS base
 
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
     build-essential \
@@ -45,33 +50,34 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins
     curl \
     git \
     make \
+    libmsgpack-dev \
     libssl-dev \
+    llvm-dev \
     g++ \
     # Needed to build VLLM & flash.
     rocthrust-dev \
     hipsparse-dev \
     hipblas-dev \
-    hipblaslt-dev \
+    hipcub-dev \
     rocblas-dev \
     hiprand-dev \
+    hipfft-dev \
     rocrand-dev \
     miopen-hip-dev \
-    hipfft-dev \
-    hipcub-dev \
     hipsolver-dev \
     rccl-dev \
     cmake \
-    python3-dev && \
+    python3.11-venv && \
     rm -rf /var/lib/apt/lists/*
 
 # Keep in sync with `server/pyproject.toml
 ARG MAMBA_VERSION=23.1.0-1
-ARG PYTORCH_VERSION='2.3.0'
-ARG ROCM_VERSION='6.0.2'
-ARG PYTHON_VERSION='3.10.10'
+ARG PYTHON_VERSION='3.11.10'
 # Automatically set by buildx
 ARG TARGETPLATFORM
-ENV PATH /opt/conda/bin:$PATH
+ENV PATH=/opt/conda/bin:$PATH
+
+ARG PYTORCH_ROCM_ARCH="gfx90a;gfx942"
 
 # TGI seem to require libssl.so.1.1 instead of libssl.so.3 so we can't use ubuntu 22.04. Ubuntu 20.04 has python==3.8, and TGI requires python>=3.9, hence the need for miniconda.
 # Install mamba
@@ -86,42 +92,141 @@ RUN chmod +x ~/mambaforge.sh && \
     mamba init && \
     rm ~/mambaforge.sh
 
+# RUN conda install intel::mkl-static intel::mkl-include
+# Install pytorch
+# On arm64 we exit with an error code
+RUN case ${TARGETPLATFORM} in \
+         "linux/arm64")  exit 1 ;; \
+         *)              /opt/conda/bin/conda update -y conda &&  \
+                         /opt/conda/bin/conda install -y "python=${PYTHON_VERSION}" ;; \
+    esac && \
+    /opt/conda/bin/conda clean -ya
+
 # Install flash-attention, torch dependencies
-RUN pip install numpy einops ninja --no-cache-dir
+RUN python3 -m pip install --upgrade pip && pip install numpy einops ninja joblib msgpack cmake --no-cache-dir && rm -rf /var/lib/apt/lists/*
 
-RUN conda install intel::mkl-static intel::mkl-include
-RUN pip uninstall -y triton && \
-    git clone --depth 1 --single-branch https://github.com/ROCm/triton.git && \
-    cd triton/python && \
-    pip install .
+RUN conda install mkl=2021
+ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/rocm/lib/:/opt/conda/lib/python3.11/site-packages/torch/lib:/opt/conda/lib/
 
-RUN git clone --depth 1 --recursive --single-branch --branch 2.3-patched https://github.com/fxmarty/pytorch.git pytorch && cd pytorch && pip install -r requirements.txt --no-cache-dir
 
-ARG _GLIBCXX_USE_CXX11_ABI="1"
-ARG CMAKE_PREFIX_PATH="/opt/conda"
+ARG COMMON_WORKDIR=/
+WORKDIR ${COMMON_WORKDIR}
+
+
+# Install HIPBLASLt
+FROM base AS build_hipblaslt
+ARG HIPBLASLT_BRANCH="e6da924"
+RUN git clone https://github.com/ROCm/hipBLASLt.git \
+    && cd hipBLASLt \
+    && git checkout ${HIPBLASLT_BRANCH} \
+    && SCCACHE_IDLE_TIMEOUT=1800 ./install.sh --architecture ${PYTORCH_ROCM_ARCH} --legacy_hipblas_direct \
+    && cd build/release \
+    && make package
+
+FROM scratch AS export_hipblaslt
+ARG COMMON_WORKDIR
+COPY --from=build_hipblaslt ${COMMON_WORKDIR}/hipBLASLt/build/release/*.deb /
+
+# RCCL build stages
+FROM base AS build_rccl
+ARG RCCL_BRANCH="rocm-6.2.0"
+RUN git clone https://github.com/ROCm/rccl \
+    && cd rccl \
+    && git checkout ${RCCL_BRANCH} \
+    && ./install.sh -p --amdgpu_targets ${PYTORCH_ROCM_ARCH}
+FROM scratch AS export_rccl
+ARG COMMON_WORKDIR
+COPY --from=build_rccl ${COMMON_WORKDIR}/rccl/build/release/*.deb /
+
+# Triton build stages
+FROM base AS build_triton
+ARG TRITON_BRANCH="e192dba"
+ARG TRITON_REPO="https://github.com/triton-lang/triton.git"
+RUN python3 -m pip install ninja cmake wheel pybind11 && git clone ${TRITON_REPO} \
+    && cd triton \
+    && git checkout ${TRITON_BRANCH} \
+    && cd python \
+    && python3 setup.py bdist_wheel --dist-dir=dist
+FROM scratch AS export_triton
+ARG COMMON_WORKDIR
+COPY --from=build_triton ${COMMON_WORKDIR}/triton/python/dist/*.whl /
+
+# # AMD-SMI build stages
+FROM base AS build_amdsmi
+RUN cd /opt/rocm/share/amd_smi \
+    && pip wheel . --wheel-dir=dist
+FROM scratch AS export_amdsmi
+COPY --from=build_amdsmi /opt/rocm/share/amd_smi/dist/*.whl /
+
+
+FROM base as build_pytorch
+
+RUN --mount=type=bind,from=export_hipblaslt,src=/,target=/install \
+    if ls /install/*.deb; then \
+        dpkg -i /install/*.deb \
+        && sed -i 's/, hipblaslt-dev \(.*\), hipcub-dev/, hipcub-dev/g' /var/lib/dpkg/status \
+        && sed -i 's/, hipblaslt \(.*\), hipfft/, hipfft/g' /var/lib/dpkg/status; \
+    fi
+
+ARG BUILD_ENVIRONMENT=pytorch-linux-jammy-rocm6.2-py3.11
 ARG PYTORCH_ROCM_ARCH="gfx90a;gfx942"
-ARG BUILD_CAFFE2="0" \
-    BUILD_CAFFE2_OPS="0" \
-    USE_CUDA="0" \
-    USE_ROCM="1" \
-    BUILD_TEST="0" \
-    USE_FBGEMM="0" \
-    USE_NNPACK="0" \
-    USE_QNNPACK="0" \
-    USE_XNNPACK="0" \
-    USE_FLASH_ATTENTION="1" \
-    USE_MEM_EFF_ATTENTION="0"
 
-RUN cd pytorch && python tools/amd_build/build_amd.py && python setup.py install
+# A commit to fix the output scaling factor issue in _scaled_mm
+# Not yet in 2.5.0-rc1
+ARG PYTORCH_BRANCH="cedc116"
+ARG PYTORCH_VISION_BRANCH="v0.19.1"
+ARG PYTORCH_REPO="https://github.com/ROCm/pytorch.git"
 
-# Set as recommended: https://github.com/ROCm/triton/wiki/A-script-to-set-program-execution-environment-in-ROCm
-ENV HIP_FORCE_DEV_KERNARG=1
+RUN git clone ${PYTORCH_REPO} pytorch \
+    && cd pytorch && git checkout ${PYTORCH_BRANCH} && git submodule update --init --recursive \
+    && pip install -r requirements.txt --no-cache-dir  \
+    && python tools/amd_build/build_amd.py \
+    && CMAKE_PREFIX_PATH=$(python3 -c 'import sys; print(sys.prefix)') python3 setup.py bdist_wheel --dist-dir=dist
+FROM scratch as export_pytorch
+ARG COMMON_WORKDIR
+COPY --from=build_pytorch ${COMMON_WORKDIR}/pytorch/dist/*.whl /
 
-# On MI250 and MI300, performances for flash with Triton FA are slightly better than CK.
-# However, Triton requires a tunning for each prompt length, which is prohibitive.
-ENV ROCM_USE_FLASH_ATTN_V2_TRITON=0
+FROM base AS install_deps
 
-FROM base AS kernel-builder
+ARG COMMON_WORKDIR
+
+# Install hipblaslt
+RUN --mount=type=bind,from=export_hipblaslt,src=/,target=/install \
+    if ls /install/*.deb; then \
+        dpkg -i /install/*.deb \
+        && sed -i 's/, hipblaslt-dev \(.*\), hipcub-dev/, hipcub-dev/g' /var/lib/dpkg/status \
+        && sed -i 's/, hipblaslt \(.*\), hipfft/, hipfft/g' /var/lib/dpkg/status; \
+    fi
+
+RUN --mount=type=bind,from=export_rccl,src=/,target=/install \
+    if ls /install/*.deb; then \
+        dpkg -i /install/*.deb \
+        # RCCL needs to be installed twice
+        && dpkg -i /install/*.deb \
+        && sed -i 's/, rccl-dev \(.*\), rocalution/, rocalution/g' /var/lib/dpkg/status \
+        && sed -i 's/, rccl \(.*\), rocalution/, rocalution/g' /var/lib/dpkg/status; \
+    fi
+
+RUN --mount=type=bind,from=export_triton,src=/,target=/install \
+    if ls /install/*.whl; then \
+        # Preemptively uninstall to prevent pip same-version no-installs
+        pip uninstall -y triton \
+        && pip install /install/*.whl; \
+    fi
+
+RUN --mount=type=bind,from=export_amdsmi,src=/,target=/install \
+    # Preemptively uninstall to prevent pip same-version no-installs
+    pip uninstall -y amdsmi \
+    && pip install /install/*.whl;
+
+RUN --mount=type=bind,from=export_pytorch,src=/,target=/install \
+    if ls /install/*.whl; then \
+        # Preemptively uninstall to prevent pip same-version no-installs
+        pip uninstall -y torch torchvision \
+        && pip install /install/*.whl; \
+    fi
+
+FROM install_deps AS kernel-builder
 
 # # Build vllm kernels
 FROM kernel-builder AS vllm-builder
@@ -142,46 +247,46 @@ COPY server/Makefile-flash-att-v2 Makefile
 RUN make build-flash-attention-v2-rocm
 
 # Build Transformers CUDA kernels (gpt-neox and bloom)
-FROM kernel-builder as custom-kernels-builder
+FROM kernel-builder AS custom-kernels-builder
 WORKDIR /usr/src
 COPY server/custom_kernels/ .
 RUN python setup.py build
 
 # Build exllama kernels
-FROM kernel-builder as exllama-kernels-builder
+FROM kernel-builder AS exllama-kernels-builder
 WORKDIR /usr/src
 COPY server/exllama_kernels/ .
 
 RUN python setup.py build
 
 # Build exllama v2 kernels
-FROM kernel-builder as exllamav2-kernels-builder
+FROM kernel-builder AS exllamav2-kernels-builder
 WORKDIR /usr/src
 COPY server/exllamav2_kernels/ .
 
 RUN python setup.py build
 
-FROM base as base-copy
+FROM install_deps AS base-copy
 
 # Text Generation Inference base env
-ENV HUGGINGFACE_HUB_CACHE=/data \
+ENV HF_HOME=/data \
     HF_HUB_ENABLE_HF_TRANSFER=1 \
     PORT=80
 
 # Copy builds artifacts from vllm builder
-COPY --from=vllm-builder /usr/src/vllm/build/lib.linux-x86_64-cpython-310 /opt/conda/lib/python3.10/site-packages
+COPY --from=vllm-builder /usr/src/vllm/build/lib.linux-x86_64-cpython-311 /opt/conda/lib/python3.11/site-packages
 
 # Copy build artifacts from flash attention v2 builder
-COPY --from=flash-att-v2-builder /usr/src/flash-attention-v2/build/lib.linux-x86_64-cpython-310 /opt/conda/lib/python3.10/site-packages
+COPY --from=flash-att-v2-builder /usr/src/flash-attention-v2/build/lib.linux-x86_64-cpython-311 /opt/conda/lib/python3.11/site-packages
 
 # Copy build artifacts from custom kernels builder
-COPY --from=custom-kernels-builder /usr/src/build/lib.linux-x86_64-cpython-310 /opt/conda/lib/python3.10/site-packages
+COPY --from=custom-kernels-builder /usr/src/build/lib.linux-x86_64-cpython-311 /opt/conda/lib/python3.11/site-packages
 
 # Copy build artifacts from exllama kernels builder
-COPY --from=exllama-kernels-builder /usr/src/build/lib.linux-x86_64-cpython-310 /opt/conda/lib/python3.10/site-packages
+COPY --from=exllama-kernels-builder /usr/src/build/lib.linux-x86_64-cpython-311 /opt/conda/lib/python3.11/site-packages
 
 # Copy build artifacts from exllamav2 kernels builder
-COPY --from=exllamav2-kernels-builder /usr/src/build/lib.linux-x86_64-cpython-310 /opt/conda/lib/python3.10/site-packages
+COPY --from=exllamav2-kernels-builder /usr/src/build/lib.linux-x86_64-cpython-311 /opt/conda/lib/python3.11/site-packages
 
 # Install server
 COPY proto proto
@@ -193,14 +298,15 @@ RUN cd server && \
     pip install ".[accelerate, peft, outlines]" --no-cache-dir
 
 # Install benchmarker
-COPY --from=builder /usr/src/target/release/text-generation-benchmark /usr/local/bin/text-generation-benchmark
+COPY --from=builder /usr/src/target/release-opt/text-generation-benchmark /usr/local/bin/text-generation-benchmark
 # Install router
-COPY --from=builder /usr/src/target/release/text-generation-router /usr/local/bin/text-generation-router
+COPY --from=builder /usr/src/target/release-opt/text-generation-router /usr/local/bin/text-generation-router
 # Install launcher
-COPY --from=builder /usr/src/target/release/text-generation-launcher /usr/local/bin/text-generation-launcher
+COPY --from=builder /usr/src/target/release-opt/text-generation-launcher /usr/local/bin/text-generation-launcher
+ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/opt/conda/lib/"
 
 # AWS Sagemaker compatible image
-FROM base as sagemaker
+FROM base AS sagemaker
 
 COPY sagemaker-entrypoint.sh entrypoint.sh
 RUN chmod +x entrypoint.sh
@@ -210,6 +316,19 @@ ENTRYPOINT ["./entrypoint.sh"]
 # Final image
 FROM base-copy
 
+# Set AS recommended: https://github.com/ROCm/triton/wiki/A-script-to-set-program-execution-environment-in-ROCm
+ENV HIP_FORCE_DEV_KERNARG=1
+
+# On MI250 and MI300, performances for flash with Triton FA are slightly better than CK.
+# However, Triton requires a tunning for each prompt length, which is prohibitive.
+ENV ROCM_USE_FLASH_ATTN_V2_TRITON=0
+ENV ROCM_USE_CUSTOM_PAGED_ATTN=1
+ENV PYTORCH_TUNABLEOP_TUNING_AFTER_WARMUP=0
+ENV VLLM_MOE_PADDING=0
+ENV ATTENTION=paged
+ENV USE_PREFIX_CACHING=0
+ENV ROCM_USE_SKINNY_GEMM=1
+
 COPY ./tgi-entrypoint.sh /tgi-entrypoint.sh
 RUN chmod +x /tgi-entrypoint.sh
 
diff --git a/Dockerfile_intel b/Dockerfile_intel
index 809992e1..7ab6bba1 100644
--- a/Dockerfile_intel
+++ b/Dockerfile_intel
@@ -1,22 +1,25 @@
-FROM lukemathwalker/cargo-chef:latest-rust-1.78 AS chef
+ARG PLATFORM=xpu
+
+FROM lukemathwalker/cargo-chef:latest-rust-1.80 AS chef
 WORKDIR /usr/src
 
 ARG CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse
 
-FROM chef as planner
+FROM chef AS planner
+COPY Cargo.lock Cargo.lock
 COPY Cargo.toml Cargo.toml
 COPY rust-toolchain.toml rust-toolchain.toml
 COPY proto proto
 COPY benchmark benchmark
 COPY router router
+COPY backends backends
 COPY launcher launcher
 RUN cargo chef prepare --recipe-path recipe.json
 
 FROM chef AS builder
 
-ARG GIT_SHA
-ARG DOCKER_LABEL
-
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+    python3.11-dev
 RUN PROTOC_ZIP=protoc-21.12-linux-x86_64.zip && \
     curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v21.12/$PROTOC_ZIP && \
     unzip -o $PROTOC_ZIP -d /usr/local bin/protoc && \
@@ -24,21 +27,52 @@ RUN PROTOC_ZIP=protoc-21.12-linux-x86_64.zip && \
     rm -f $PROTOC_ZIP
 
 COPY --from=planner /usr/src/recipe.json recipe.json
-RUN cargo chef cook --release --recipe-path recipe.json
+RUN cargo chef cook --profile release-opt --recipe-path recipe.json
+
+ARG GIT_SHA
+ARG DOCKER_LABEL
 
 COPY Cargo.toml Cargo.toml
 COPY rust-toolchain.toml rust-toolchain.toml
 COPY proto proto
 COPY benchmark benchmark
 COPY router router
+COPY backends backends
 COPY launcher launcher
-RUN cargo build --release
+RUN cargo build --profile release-opt
 
 
 # Text Generation Inference base image for Intel
-FROM intel/intel-extension-for-pytorch:2.1.30-xpu as base
+
+FROM intel/intel-extension-for-pytorch:2.3.110-xpu AS xpu
 
 USER root
+
+ARG MAMBA_VERSION=23.1.0-1
+ARG PYTHON_VERSION='3.11.10'
+# Automatically set by buildx
+ARG TARGETPLATFORM
+ENV PATH /opt/conda/bin:$PATH
+
+# TGI seem to require libssl.so.1.1 instead of libssl.so.3 so we can't use ubuntu 22.04. Ubuntu 20.04 has python==3.8, and TGI requires python>=3.9, hence the need for miniconda.
+# Install mamba
+# translating Docker's TARGETPLATFORM into mamba arches
+RUN case ${TARGETPLATFORM} in \
+         "linux/arm64")  MAMBA_ARCH=aarch64  ;; \
+         *)              MAMBA_ARCH=x86_64   ;; \
+    esac && \
+    curl -fsSL -v -o ~/mambaforge.sh -O  "https://github.com/conda-forge/miniforge/releases/download/${MAMBA_VERSION}/Mambaforge-${MAMBA_VERSION}-Linux-${MAMBA_ARCH}.sh"
+RUN chmod +x ~/mambaforge.sh && \
+    bash ~/mambaforge.sh -b -p /opt/conda && \
+    rm ~/mambaforge.sh
+
+RUN case ${TARGETPLATFORM} in \
+         "linux/arm64")  exit 1 ;; \
+         *)              /opt/conda/bin/conda update -y conda &&  \
+                         /opt/conda/bin/conda install -y "python=${PYTHON_VERSION}" ;; \
+    esac && \
+    /opt/conda/bin/conda clean -ya
+
 # libssl.so.1.1 is not installed on Ubuntu 22.04 by default, install it
 RUN wget http://nz2.archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_amd64.deb && \
     dpkg -i ./libssl1.1_1.1.1f-1ubuntu2_amd64.deb
@@ -48,17 +82,16 @@ RUN wget -qO - https://repositories.intel.com/gpu/intel-graphics.key | gpg --dea
 RUN wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB \
 | gpg --dearmor | tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null && echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | tee /etc/apt/sources.list.d/oneAPI.list
 
-RUN apt-get update && apt install -y intel-basekit xpu-smi
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt install -y intel-basekit xpu-smi cmake ninja-build pciutils
 
 # Text Generation Inference base env
-ENV HUGGINGFACE_HUB_CACHE=/data \
+ENV HF_HOME=/data \
     HF_HUB_ENABLE_HF_TRANSFER=1 \
     PORT=80
 
 
 WORKDIR /usr/src
-RUN wget https://intel-extension-for-pytorch.s3.amazonaws.com/ipex_dev/xpu/intel_extension_for_pytorch-2.1.30a0-cp310-cp310-linux_x86_64.whl
-RUN pip install intel_extension_for_pytorch-2.1.30a0-cp310-cp310-linux_x86_64.whl
+RUN pip install torch==2.3.1+cxx11.abi torchvision==0.18.1+cxx11.abi torchaudio==2.3.1+cxx11.abi intel-extension-for-pytorch==2.3.110+xpu oneccl_bind_pt==2.3.100+xpu --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ --no-cache-dir
 
 # Install server
 COPY proto proto
@@ -66,26 +99,112 @@ COPY server server
 COPY server/Makefile server/Makefile
 RUN cd server && \
     make gen-server && \
-    pip install -r requirements_cuda.txt && \
+    pip install -r requirements_intel.txt && \
     pip install ".[accelerate, peft, outlines]" --no-cache-dir
 
 ENV CCL_ROOT=/opt/intel/oneapi/ccl/latest
 ENV I_MPI_ROOT=/opt/intel/oneapi/mpi/latest
 ENV FI_PROVIDER_PATH=/opt/intel/oneapi/mpi/latest/opt/mpi/libfabric/lib/prov:/usr/lib/x86_64-linux-gnu/libfabric
 ENV LIBRARY_PATH=/opt/intel/oneapi/mpi/latest/lib:/opt/intel/oneapi/ccl/latest/lib/:/opt/intel/oneapi/mkl/latest/lib/:/opt/intel/oneapi/compiler/latest/lib
-ENV LD_LIBRARY_PATH=/opt/intel/oneapi/ccl/latest/lib/:/opt/intel/oneapi/mpi/latest/opt/mpi/libfabric/lib:/opt/intel/oneapi/mpi/latest/lib:/opt/intel/oneapi/mkl/latest/lib:/opt/intel/oneapi/compiler/latest/opt/compiler/lib:/opt/intel/oneapi/compiler/latest/lib:/opt/intel/oneapi/lib:/opt/intel/oneapi/lib/intel64:
-ENV PATH=/opt/intel/oneapi/mpi/latest/opt/mpi/libfabric/bin:/opt/intel/oneapi/mpi/latest/bin:/opt/intel/oneapi/mpi/latest/opt/mpi/libfabric/bin:/opt/intel/oneapi/mkl/latest/bin/:/opt/intel/oneapi/compiler/latest/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+ENV LD_LIBRARY_PATH=/opt/intel/oneapi/ccl/latest/lib/:/opt/intel/oneapi/mpi/latest/opt/mpi/libfabric/lib:/opt/intel/oneapi/mpi/latest/lib:/opt/intel/oneapi/mkl/latest/lib:/opt/intel/oneapi/compiler/latest/opt/compiler/lib:/opt/intel/oneapi/compiler/latest/lib:/opt/intel/oneapi/lib:/opt/intel/oneapi/lib/intel64:/opt/conda/lib
+ENV PATH=/opt/conda/bin:/opt/intel/oneapi/mpi/latest/opt/mpi/libfabric/bin:/opt/intel/oneapi/mpi/latest/bin:/opt/intel/oneapi/mpi/latest/opt/mpi/libfabric/bin:/opt/intel/oneapi/mkl/latest/bin/:/opt/intel/oneapi/compiler/latest/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
 ENV CCL_ZE_IPC_EXCHANGE=sockets
+ENV CMAKE_PREFIX_PATH=/opt/intel/oneapi/mkl/latest/lib/cmake:/opt/intel/oneapi/compiler/latest
+ENV CPATH=/opt/intel/oneapi/mpi/latest/include:/opt/intel/oneapi/ccl/latest/include:/opt/intel/oneapi/mkl/latest/include
 
 # Install benchmarker
-COPY --from=builder /usr/src/target/release/text-generation-benchmark /usr/local/bin/text-generation-benchmark
+COPY --from=builder /usr/src/target/release-opt/text-generation-benchmark /usr/local/bin/text-generation-benchmark
 # Install router
-COPY --from=builder /usr/src/target/release/text-generation-router /usr/local/bin/text-generation-router
+COPY --from=builder /usr/src/target/release-opt/text-generation-router /usr/local/bin/text-generation-router
 # Install launcher
-COPY --from=builder /usr/src/target/release/text-generation-launcher /usr/local/bin/text-generation-launcher
+COPY --from=builder /usr/src/target/release-opt/text-generation-launcher /usr/local/bin/text-generation-launcher
 
-# Final image
-FROM base
 
+# Text Generation Inference base image for Intel-cpu
+FROM ubuntu:22.04 AS cpu
+
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+    curl \
+    ca-certificates \
+    make \
+    g++ \
+    git \
+    wget \
+    cmake \
+    libnuma-dev
+
+ENV HUGGINGFACE_HUB_CACHE=/data \
+    HF_HUB_ENABLE_HF_TRANSFER=1 \
+    PORT=80
+
+ARG MAMBA_VERSION=23.1.0-1
+ARG PYTHON_VERSION='3.11.10'
+# Automatically set by buildx
+ARG TARGETPLATFORM
+ENV PATH /opt/conda/bin:$PATH
+
+# TGI seem to require libssl.so.1.1 instead of libssl.so.3 so we can't use ubuntu 22.04. Ubuntu 20.04 has python==3.8, and TGI requires python>=3.9, hence the need for miniconda.
+# Install mamba
+# translating Docker's TARGETPLATFORM into mamba arches
+RUN case ${TARGETPLATFORM} in \
+         "linux/arm64")  MAMBA_ARCH=aarch64  ;; \
+         *)              MAMBA_ARCH=x86_64   ;; \
+    esac && \
+    curl -fsSL -v -o ~/mambaforge.sh -O  "https://github.com/conda-forge/miniforge/releases/download/${MAMBA_VERSION}/Mambaforge-${MAMBA_VERSION}-Linux-${MAMBA_ARCH}.sh"
+RUN chmod +x ~/mambaforge.sh && \
+    bash ~/mambaforge.sh -b -p /opt/conda && \
+    rm ~/mambaforge.sh
+
+RUN case ${TARGETPLATFORM} in \
+         "linux/arm64")  exit 1 ;; \
+         *)              /opt/conda/bin/conda update -y conda &&  \
+                         /opt/conda/bin/conda install -y "python=${PYTHON_VERSION}" ;; \
+    esac && \
+    /opt/conda/bin/conda clean -ya
+
+RUN conda install -c conda-forge gperftools mkl
+
+RUN pip install https://download.pytorch.org/whl/nightly/cpu/torch-2.4.0.dev20240612%2Bcpu-cp311-cp311-linux_x86_64.whl
+RUN pip install https://download.pytorch.org/whl/nightly/cpu/torchvision-0.19.0.dev20240612%2Bcpu-cp311-cp311-linux_x86_64.whl
+RUN pip install https://download.pytorch.org/whl/nightly/cpu/torchaudio-2.4.0.dev20240612%2Bcpu-cp311-cp311-linux_x86_64.whl
+RUN pip install triton py-libnuma
+
+WORKDIR /usr/src
+
+RUN git clone https://github.com/intel/intel-extension-for-pytorch && cd intel-extension-for-pytorch && git checkout eda7a7c42df6f9a64e0de9c2b69304ee02f2c32a
+
+RUN git clone https://github.com/intel/torch-ccl.git && cd torch-ccl && git checkout ccl_torch_dev_0131
+
+RUN cd intel-extension-for-pytorch && git submodule sync && git submodule update --init --recursive && python setup.py install
+
+RUN cd torch-ccl && git submodule sync && git submodule update --init --recursive && pip install .
+
+ENV LD_PRELOAD=/opt/conda/lib/libtcmalloc.so
+ENV CCL_ROOT=/opt/conda/lib/python3.11/site-packages/oneccl_bindings_for_pytorch
+ENV I_MPI_ROOT=/opt/conda/lib/python3.11/site-packages/oneccl_bindings_for_pytorch
+ENV FI_PROVIDER_PATH=/opt/conda/lib/python3.11/site-packages/oneccl_bindings_for_pytorch/opt/mpi/libfabric/lib/prov:/usr/lib64/libfabric
+ENV LD_LIBRARY_PATH=/opt/conda/lib/python3.11/site-packages/oneccl_bindings_for_pytorch/opt/mpi/libfabric/lib:/opt/conda/lib/python3.11/site-packages/oneccl_bindings_for_pytorch/lib
+ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/opt/conda/lib/"
+
+# Install server
+COPY proto proto
+COPY server server
+COPY server/Makefile server/Makefile
+RUN cd server && \
+    make gen-server && \
+    pip install -r requirements_intel.txt && \
+    pip install ".[accelerate, peft, outlines]" --no-cache-dir
+
+# Install benchmarker
+COPY --from=builder /usr/src/target/release-opt/text-generation-benchmark /usr/local/bin/text-generation-benchmark
+# Install router
+COPY --from=builder /usr/src/target/release-opt/text-generation-router /usr/local/bin/text-generation-router
+# Install launcher
+COPY --from=builder /usr/src/target/release-opt/text-generation-launcher /usr/local/bin/text-generation-launcher
+
+FROM ${PLATFORM} AS final
+ENV ATTENTION=paged
+ENV USE_PREFIX_CACHING=0
+ENV CUDA_GRAPHS=0
 ENTRYPOINT ["text-generation-launcher"]
 CMD ["--json-output"]
diff --git a/Makefile b/Makefile
index 96e67e2b..664b869d 100644
--- a/Makefile
+++ b/Makefile
@@ -1,20 +1,22 @@
 install-server:
 	cd server && make install
 
-install-integration-tests:
-	cd integration-tests && pip install -r requirements.txt
-	cd clients/python && pip install .
+install-server-cpu:
+	cd server && make install-server
 
 install-router:
-	cd router && cargo install --locked --path .
+	cargo install --path backends/v3/
 
 install-launcher:
-	cd launcher && cargo install --locked --path .
+	cargo install --path launcher/
 
 install-benchmark:
-	cd benchmark && cargo install --locked --path .
+	cargo install --path benchmark/
 
-install: install-server install-router install-launcher install-custom-kernels
+install: install-server install-router install-launcher
+
+
+install-cpu: install-server-cpu install-router install-launcher
 
 server-dev:
 	cd server && make run-dev
@@ -25,6 +27,10 @@ router-dev:
 rust-tests: install-router install-launcher
 	cargo test
 
+install-integration-tests:
+	cd integration-tests && pip install -r requirements.txt
+	cd clients/python && pip install .
+
 integration-tests: install-integration-tests
 	pytest -s -vv -m "not private" integration-tests
 
@@ -44,6 +50,3 @@ run-falcon-7b-instruct:
 
 clean:
 	rm -rf target aml
-
-debug_image_build:
-	docker build --no-cache --progress=plain -t debug_tgi .
diff --git a/router/client/Cargo.toml b/backends/client/Cargo.toml
similarity index 89%
rename from router/client/Cargo.toml
rename to backends/client/Cargo.toml
index bc4ae72e..db423c4b 100644
--- a/router/client/Cargo.toml
+++ b/backends/client/Cargo.toml
@@ -6,10 +6,11 @@ authors.workspace = true
 homepage.workspace = true
 
 [dependencies]
+async-trait = "^0.1"
+base64 = { workspace = true }
 futures = "^0.3"
 grpc-metadata = { path = "../grpc-metadata" }
 prost = "^0.12"
-rand = "0.8.5"
 thiserror = "^1.0"
 tokio = { version = "^1.32", features = ["sync"] }
 tonic = "^0.10"
diff --git a/backends/client/build.rs b/backends/client/build.rs
new file mode 100644
index 00000000..210cd603
--- /dev/null
+++ b/backends/client/build.rs
@@ -0,0 +1,35 @@
+use std::fs;
+
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    println!("cargo:rerun-if-changed=../../proto/");
+
+    fs::create_dir_all("src/v2/pb").unwrap_or(());
+    let mut config = prost_build::Config::new();
+    config.protoc_arg("--experimental_allow_proto3_optional");
+
+    tonic_build::configure()
+        .build_client(true)
+        .build_server(false)
+        .out_dir("src/v2/pb")
+        .include_file("mod.rs")
+        .compile_with_config(config, &["../../proto/generate.proto"], &["../../proto"])
+        .map_err(|e| match e.kind(){
+            std::io::ErrorKind::NotFound => {panic!("`protoc` not found, install libprotoc")},
+            std::io::ErrorKind::Other => {panic!("`protoc` version unsupported, upgrade protoc: https://github.com/protocolbuffers/protobuf/releases")},
+            e => {e}
+        }).unwrap_or_else(|e| panic!("protobuf compilation failed: {e}"));
+
+    fs::create_dir_all("src/v3/pb").unwrap_or(());
+    let mut config = prost_build::Config::new();
+    config.protoc_arg("--experimental_allow_proto3_optional");
+
+    tonic_build::configure()
+        .build_client(true)
+        .build_server(false)
+        .out_dir("src/v3/pb")
+        .include_file("mod.rs")
+        .compile_with_config(config, &["../../proto/v3/generate.proto"], &["../../proto"])
+        .unwrap_or_else(|e| panic!("protobuf compilation failed: {e}"));
+
+    Ok(())
+}
diff --git a/backends/client/src/lib.rs b/backends/client/src/lib.rs
new file mode 100644
index 00000000..45bee10c
--- /dev/null
+++ b/backends/client/src/lib.rs
@@ -0,0 +1,91 @@
+//! Text Generation gRPC client library
+
+pub mod v2;
+pub mod v3;
+
+use async_trait::async_trait;
+use base64::{engine::general_purpose::STANDARD, Engine};
+use thiserror::Error;
+use tonic::transport;
+use tonic::Status;
+
+pub use v3::{Chunk, Image, Input, InputChunk};
+
+#[async_trait]
+pub trait Health {
+    /// Check if a generate server is healthy by asking it to allocate a tensor on device
+    async fn device_health(&self) -> Result<()>;
+
+    /// Check if a generate server is healthy by doing a forward pass.
+    /// EXPENSIVE
+    async fn model_health(&self) -> Result<()>;
+}
+
+#[derive(Debug)]
+pub struct ShardInfo {
+    pub requires_padding: bool,
+    pub dtype: String,
+    pub device_type: String,
+    pub window_size: Option<u32>,
+    pub speculate: u32,
+}
+
+#[derive(Error, Debug, Clone)]
+pub enum ClientError {
+    #[error("Could not connect to Text Generation server: {0}")]
+    Connection(String),
+    #[error("Server error: {0}")]
+    Generation(String),
+    #[error("Sharded results are empty")]
+    EmptyResults,
+}
+
+impl From<Status> for ClientError {
+    fn from(err: Status) -> Self {
+        let err = Self::Generation(err.message().to_string());
+        tracing::error!("{err}");
+        err
+    }
+}
+
+impl From<transport::Error> for ClientError {
+    fn from(err: transport::Error) -> Self {
+        let err = Self::Connection(err.to_string());
+        tracing::error!("{err}");
+        err
+    }
+}
+
+// Small convenience re-wrapping of `Chunk`.
+impl From<Chunk> for InputChunk {
+    fn from(chunk: Chunk) -> Self {
+        InputChunk { chunk: Some(chunk) }
+    }
+}
+
+/// Convert input chunks to a stringly-typed input for backwards
+/// compat for backends that haven't implemented chunked inputs.
+pub trait ChunksToString {
+    /// Convert chunks to string.
+    fn chunks_to_string(&self) -> String;
+}
+
+impl ChunksToString for Vec<InputChunk> {
+    fn chunks_to_string(&self) -> String {
+        let mut output = String::new();
+        self.iter().for_each(|c| match &c.chunk {
+            Some(Chunk::Text(text)) => output.push_str(text),
+            Some(Chunk::Image(Image { data, mimetype })) => {
+                let encoded = STANDARD.encode(data);
+                output.push_str(&format!("![](data:{};base64,{})", mimetype, encoded))
+            }
+            // We don't create empty chunks, so this should be unreachable.
+            None => unreachable!("Chunks should never be empty"),
+        });
+        output
+    }
+}
+
+static WARMUP_IMAGE_BASE64 :&str = "iVBORw0KGgoAAAANSUhEUgAAABQAAAAUCAIAAAAC64paAAABg2lDQ1BJQ0MgcHJvZmlsZQAAKJF9kT1Iw0AcxV/TSotUROxQxCFDdbKLijjWKhShQqgVWnUwufQLmrQkKS6OgmvBwY/FqoOLs64OroIg+AHi7OCk6CIl/i8ptIjx4Lgf7+497t4BQqvKNDOQADTdMjKppJjLr4rBVwQQwhAERGVm1uckKQ3P8XUPH1/v4jzL+9yfY0AtmAzwicQJVjcs4g3imU2rznmfOMLKskp8Tjxh0AWJH7muuPzGueSwwDMjRjYzTxwhFks9rPQwKxsa8TRxTNV0yhdyLquctzhr1Qbr3JO/MFzQV5a5TnMUKSxiCRJEKGiggiosxGnVSTGRof2kh3/E8UvkUshVASPHAmrQIDt+8D/43a1ZnJp0k8JJoO/Ftj/GgOAu0G7a9vexbbdPAP8zcKV3/bUWMPtJerOrxY6AwW3g4rqrKXvA5Q4QfarLhuxIfppCsQi8n9E35YHhW6B/ze2ts4/TByBLXaVvgINDYLxE2ese7w719vbvmU5/PycecohsjayNAAAACXBIWXMAAC4jAAAuIwF4pT92AAAAB3RJTUUH6AQIEQMnlTSSjwAAABl0RVh0Q29tbWVudABDcmVhdGVkIHdpdGggR0lNUFeBDhcAAAASSURBVDjLY2AYBaNgFIyCoQsABMQAAeRw1DoAAAAASUVORK5CYII=";
+
+pub type Result<T> = std::result::Result<T, ClientError>;
diff --git a/backends/client/src/v2/client.rs b/backends/client/src/v2/client.rs
new file mode 100644
index 00000000..7a6922f0
--- /dev/null
+++ b/backends/client/src/v2/client.rs
@@ -0,0 +1,260 @@
+/// Single shard Client
+use crate::v2::pb;
+use crate::{ClientError, Result};
+
+use crate::WARMUP_IMAGE_BASE64;
+use grpc_metadata::InjectTelemetryContext;
+use pb::generate::v2::text_generation_service_client::TextGenerationServiceClient;
+use pb::generate::v2::*;
+use std::cmp::min;
+use std::time::Duration;
+use tonic::transport::{Channel, Uri};
+use tracing::instrument;
+
+/// Text Generation Inference gRPC client
+#[derive(Debug, Clone)]
+pub struct Client {
+    stub: TextGenerationServiceClient<Channel>,
+}
+
+impl Client {
+    /// Returns a client connected to the given url
+    pub async fn connect(uri: Uri) -> Result<Self> {
+        let channel = Channel::builder(uri).connect().await?;
+
+        Ok(Self {
+            stub: TextGenerationServiceClient::new(channel),
+        })
+    }
+
+    /// Returns a client connected to the given unix socket
+    pub async fn connect_uds(path: String) -> Result<Self> {
+        let channel = Channel::from_shared("http://[::]:50051".to_string())
+            .unwrap()
+            .connect_with_connector(tower::service_fn(move |_: Uri| {
+                tokio::net::UnixStream::connect(path.clone())
+            }))
+            .await?;
+
+        Ok(Self {
+            stub: TextGenerationServiceClient::new(channel),
+        })
+    }
+
+    /// Returns a list of uris or unix sockets of all shards
+    #[instrument(skip(self))]
+    pub async fn service_discovery(&mut self) -> Result<Vec<String>> {
+        let request = tonic::Request::new(ServiceDiscoveryRequest {}).inject_context();
+        let response = self.stub.service_discovery(request).await.map_err(|_| {
+            ClientError::Connection("Server does not support v2 interface".to_string())
+        })?;
+        let urls = response
+            .into_inner()
+            .urls
+            .into_iter()
+            // Remove unix socket prefix
+            .map(|url| match url.strip_prefix("unix://") {
+                None => url,
+                Some(stripped_url) => stripped_url.to_string(),
+            })
+            .collect();
+        Ok(urls)
+    }
+
+    /// Get model info
+    #[instrument(skip(self))]
+    pub async fn info(&mut self) -> Result<InfoResponse> {
+        let request = tonic::Request::new(InfoRequest {}).inject_context();
+        let response = self.stub.info(request).await?.into_inner();
+        Ok(response)
+    }
+
+    /// Get model health
+    #[instrument(skip(self))]
+    pub async fn health(&mut self) -> Result<HealthResponse> {
+        let request = tonic::Request::new(HealthRequest {}).inject_context();
+        let response = self.stub.health(request).await?.into_inner();
+        Ok(response)
+    }
+
+    /// Clear the past generations cache
+    #[instrument(skip(self))]
+    pub async fn clear_cache(&mut self, batch_id: Option<u64>) -> Result<()> {
+        let request = tonic::Request::new(ClearCacheRequest { id: batch_id }).inject_context();
+        self.stub.clear_cache(request).await?;
+        Ok(())
+    }
+
+    /// Filter a cached batch
+    #[instrument(skip(self))]
+    pub async fn filter_batch(
+        &mut self,
+        batch_id: u64,
+        request_ids: Vec<u64>,
+    ) -> Result<Option<CachedBatch>> {
+        let request = tonic::Request::new(FilterBatchRequest {
+            batch_id,
+            request_ids,
+        })
+        .inject_context();
+        let filtered_batch = self.stub.filter_batch(request).await?.into_inner();
+        Ok(filtered_batch.batch)
+    }
+
+    /// Warmup on a max size batch
+    ///
+    /// Returns the maximum amount of tokens supported by the hardware
+    #[instrument(skip_all)]
+    pub async fn warmup(
+        &mut self,
+        max_input_length: u32,
+        max_prefill_tokens: u32,
+        max_total_tokens: u32,
+        max_batch_total_tokens: u32,
+        max_batch_size: Option<usize>,
+    ) -> Result<Option<u32>> {
+        let mut n_tokens = 0;
+        let mut requests = Vec::new();
+        // Create requests
+        while n_tokens < max_prefill_tokens {
+            let truncate = min(max_input_length, max_prefill_tokens - n_tokens);
+
+            let mut inputs = String::new();
+            inputs.push_str(&"_test ".to_string().repeat(max_input_length as usize));
+            if n_tokens == 0 {
+                // 1 request is enough to test vision heads.
+                // Sending images on other queries messes up easily with truncation.
+                inputs.push_str(&format!(
+                    "![](data:image/jpeg;base64,{WARMUP_IMAGE_BASE64})",
+                ));
+            }
+
+            requests.push(Request {
+                id: 0,
+                inputs,
+                // We truncate the input on the server side to be sure that it has the correct size
+                truncate,
+                // Set sampling parameters to also take these ops into account in the max memory
+                parameters: Some(NextTokenChooserParameters {
+                    temperature: 0.9,
+                    top_k: 10,
+                    top_p: 0.9,
+                    typical_p: 0.9,
+                    do_sample: false,
+                    seed: 0,
+                    repetition_penalty: 1.2,
+                    frequency_penalty: 0.1,
+                    watermark: true,
+                    grammar: String::new(),
+                    grammar_type: GrammarType::None as i32,
+                }),
+                stopping_parameters: Some(StoppingCriteriaParameters {
+                    max_new_tokens: max_total_tokens - truncate,
+                    stop_sequences: vec![],
+                    ignore_eos_token: true,
+                }),
+                prefill_logprobs: true,
+                top_n_tokens: 20,
+            });
+            n_tokens += max_input_length;
+
+            // Check max_batch_size
+            if Some(requests.len()) == max_batch_size {
+                break;
+            }
+        }
+
+        let batch = Batch {
+            id: 0,
+            size: requests.len() as u32,
+            requests,
+            max_tokens: 0,
+        };
+
+        let request = tonic::Request::new(WarmupRequest {
+            batch: Some(batch),
+            max_input_length,
+            max_prefill_tokens,
+            max_total_tokens,
+            max_batch_total_tokens,
+        })
+        .inject_context();
+        let response = self.stub.warmup(request).await?.into_inner();
+        Ok(response.max_supported_total_tokens)
+    }
+
+    /// Generate one token for each request in the given batch
+    ///
+    /// Returns Generation for each request in batch
+    /// and the next cached batch
+    #[instrument(skip_all, fields(id = &batch.id, size = &batch.size))]
+    pub async fn prefill(
+        &mut self,
+        batch: Batch,
+    ) -> Result<(Vec<Generation>, Option<CachedBatch>, PrefillTimings)> {
+        let request = tonic::Request::new(PrefillRequest { batch: Some(batch) }).inject_context();
+        let response = self.stub.prefill(request).await?.into_inner();
+        Ok((
+            response.generations,
+            response.batch,
+            PrefillTimings::new(response.forward_ns, response.decode_ns, response.total_ns),
+        ))
+    }
+
+    /// Generate one token for each request in the given cached batches
+    ///
+    /// Returns Generation for each request in batches
+    /// and the next cached batch
+    #[instrument(skip_all, fields(size = batches.iter().map(|batch|{batch.size}).sum::<u32>()))]
+    pub async fn decode(
+        &mut self,
+        batches: Vec<CachedBatch>,
+    ) -> Result<(Vec<Generation>, Option<CachedBatch>, DecodeTimings)> {
+        let request = tonic::Request::new(DecodeRequest { batches }).inject_context();
+        let response = self.stub.decode(request).await?.into_inner();
+        Ok((
+            response.generations,
+            response.batch,
+            DecodeTimings::new(
+                response.concat_ns,
+                response.forward_ns,
+                response.decode_ns,
+                response.total_ns,
+            ),
+        ))
+    }
+}
+
+pub struct PrefillTimings {
+    pub forward: Duration,
+    pub decode: Duration,
+    pub total: Duration,
+}
+
+impl PrefillTimings {
+    fn new(forward_ns: u64, decode_ns: u64, total_ns: u64) -> Self {
+        Self {
+            forward: Duration::from_nanos(forward_ns),
+            decode: Duration::from_nanos(decode_ns),
+            total: Duration::from_nanos(total_ns),
+        }
+    }
+}
+
+pub struct DecodeTimings {
+    pub concat: Option<Duration>,
+    pub forward: Duration,
+    pub decode: Duration,
+    pub total: Duration,
+}
+
+impl DecodeTimings {
+    fn new(concat_ns: Option<u64>, forward_ns: u64, decode_ns: u64, total_ns: u64) -> Self {
+        Self {
+            concat: concat_ns.map(Duration::from_nanos),
+            forward: Duration::from_nanos(forward_ns),
+            decode: Duration::from_nanos(decode_ns),
+            total: Duration::from_nanos(total_ns),
+        }
+    }
+}
diff --git a/backends/client/src/v2/mod.rs b/backends/client/src/v2/mod.rs
new file mode 100644
index 00000000..6b14b9f3
--- /dev/null
+++ b/backends/client/src/v2/mod.rs
@@ -0,0 +1,13 @@
+#[allow(clippy::derive_partial_eq_without_eq)]
+mod pb;
+
+mod client;
+mod sharded_client;
+
+pub use client::Client;
+pub use pb::generate::v2::HealthResponse;
+pub use pb::generate::v2::{
+    Batch, CachedBatch, FinishReason, GeneratedText, Generation, GrammarType, InfoResponse,
+    NextTokenChooserParameters, Request, StoppingCriteriaParameters, Tokens,
+};
+pub use sharded_client::ShardedClient;
diff --git a/router/client/src/sharded_client.rs b/backends/client/src/v2/sharded_client.rs
similarity index 75%
rename from router/client/src/sharded_client.rs
rename to backends/client/src/v2/sharded_client.rs
index fdd84035..2709ea88 100644
--- a/router/client/src/sharded_client.rs
+++ b/backends/client/src/v2/sharded_client.rs
@@ -1,12 +1,17 @@
-/// Copyright (C) 2024 Habana Labs, Ltd. an Intel Company.
-
-use crate::client::{DecodeTimings, PrefillTimings};
 /// Multi shard Client
-use crate::{Batch, CachedBatch, Client, Generation, HealthResponse, ShardInfo};
+use crate::{v2, Health, ShardInfo};
 use crate::{ClientError, Result};
+
+use crate::v2::InfoResponse;
+use async_trait::async_trait;
 use futures::future::join_all;
 use tonic::transport::Uri;
 use tracing::instrument;
+use v2::client::{DecodeTimings, PrefillTimings};
+use v2::{
+    Batch, CachedBatch, Client, Generation, GrammarType, HealthResponse,
+    NextTokenChooserParameters, Request, StoppingCriteriaParameters,
+};
 
 #[derive(Debug, Clone)]
 /// Text Generation Inference gRPC multi client
@@ -49,7 +54,7 @@ impl ShardedClient {
             .iter_mut()
             .map(|client| client.info())
             .collect();
-        join_all(futures).await.pop().unwrap()
+        join_all(futures).await.pop().unwrap().map(ShardInfo::from)
     }
 
     /// GRPC health check
@@ -99,8 +104,8 @@ impl ShardedClient {
         max_input_length: u32,
         max_prefill_tokens: u32,
         max_total_tokens: u32,
+        max_batch_total_tokens: u32,
         max_batch_size: Option<usize>,
-        model_id: &str,
     ) -> Result<Option<u32>> {
         let futures: Vec<_> = self
             .clients
@@ -110,8 +115,8 @@ impl ShardedClient {
                     max_input_length,
                     max_prefill_tokens,
                     max_total_tokens,
+                    max_batch_total_tokens,
                     max_batch_size,
-                    model_id
                 ))
             })
             .collect();
@@ -189,3 +194,60 @@ impl ShardedClient {
         Ok((generations, next_batch, timings))
     }
 }
+
+impl From<InfoResponse> for ShardInfo {
+    fn from(value: InfoResponse) -> Self {
+        Self {
+            requires_padding: value.requires_padding,
+            dtype: value.dtype,
+            device_type: value.device_type,
+            window_size: value.window_size,
+            speculate: value.speculate,
+        }
+    }
+}
+
+#[async_trait]
+impl Health for ShardedClient {
+    async fn device_health(&self) -> Result<()> {
+        self.clone().health().await?;
+        Ok(())
+    }
+
+    async fn model_health(&self) -> Result<()> {
+        // Dummy batch of 1 token and 1 generated token
+        let liveness_request = Request {
+            id: u64::MAX,
+            inputs: "liveness".to_string(),
+            truncate: 10,
+            prefill_logprobs: false,
+            parameters: Some(NextTokenChooserParameters {
+                temperature: 1.0,
+                top_k: 0,
+                top_p: 1.0,
+                typical_p: 1.0,
+                do_sample: false,
+                seed: 0,
+                repetition_penalty: 1.0,
+                frequency_penalty: 0.0,
+                watermark: false,
+                grammar: String::new(),
+                grammar_type: GrammarType::None as i32,
+            }),
+            stopping_parameters: Some(StoppingCriteriaParameters {
+                max_new_tokens: 1,
+                stop_sequences: vec![],
+                ignore_eos_token: false,
+            }),
+            top_n_tokens: 0,
+        };
+        let batch = Batch {
+            id: u64::MAX,
+            requests: vec![liveness_request],
+            size: 1,
+            max_tokens: 2,
+        };
+        self.clone().prefill(batch).await?;
+        Ok(())
+    }
+}
diff --git a/backends/client/src/v3/client.rs b/backends/client/src/v3/client.rs
new file mode 100644
index 00000000..6274c359
--- /dev/null
+++ b/backends/client/src/v3/client.rs
@@ -0,0 +1,288 @@
+use crate::v3::{pb, Chunk};
+use crate::{ClientError, Result, WARMUP_IMAGE_BASE64};
+/// Single shard Client
+use base64::engine::general_purpose::STANDARD;
+use base64::Engine;
+use grpc_metadata::InjectTelemetryContext;
+use pb::generate::v3::text_generation_service_client::TextGenerationServiceClient;
+use pb::generate::v3::*;
+use std::cmp::min;
+use std::time::Duration;
+use tonic::transport::{Channel, Uri};
+use tracing::instrument;
+
+/// Text Generation Inference gRPC client
+#[derive(Debug, Clone)]
+pub struct Client {
+    stub: TextGenerationServiceClient<Channel>,
+}
+
+impl Client {
+    /// Returns a client connected to the given url
+    pub async fn connect(uri: Uri) -> Result<Self> {
+        let channel = Channel::builder(uri).connect().await?;
+
+        Ok(Self {
+            stub: TextGenerationServiceClient::new(channel),
+        })
+    }
+
+    /// Returns a client connected to the given unix socket
+    pub async fn connect_uds(path: String) -> Result<Self> {
+        let channel = Channel::from_shared("http://[::]:50051".to_string())
+            .unwrap()
+            .connect_with_connector(tower::service_fn(move |_: Uri| {
+                tokio::net::UnixStream::connect(path.clone())
+            }))
+            .await?;
+
+        Ok(Self {
+            stub: TextGenerationServiceClient::new(channel),
+        })
+    }
+
+    /// Returns a list of uris or unix sockets of all shards
+    #[instrument(skip(self))]
+    pub async fn service_discovery(&mut self) -> Result<Vec<String>> {
+        let request = tonic::Request::new(ServiceDiscoveryRequest {}).inject_context();
+        let response = self.stub.service_discovery(request).await.map_err(|_| {
+            ClientError::Connection("Server does not support v3 interface".to_string())
+        })?;
+        let urls = response
+            .into_inner()
+            .urls
+            .into_iter()
+            // Remove unix socket prefix
+            .map(|url| match url.strip_prefix("unix://") {
+                None => url,
+                Some(stripped_url) => stripped_url.to_string(),
+            })
+            .collect();
+        Ok(urls)
+    }
+
+    /// Get model info
+    #[instrument(skip(self))]
+    pub async fn info(&mut self) -> Result<InfoResponse> {
+        let request = tonic::Request::new(InfoRequest {}).inject_context();
+        let response = self.stub.info(request).await?.into_inner();
+        Ok(response)
+    }
+
+    /// Get model health
+    #[instrument(skip(self))]
+    pub async fn health(&mut self) -> Result<HealthResponse> {
+        let request = tonic::Request::new(HealthRequest {}).inject_context();
+        let response = self.stub.health(request).await?.into_inner();
+        Ok(response)
+    }
+
+    /// Clear the past generations cache
+    #[instrument(skip(self))]
+    pub async fn clear_cache(&mut self, batch_id: Option<u64>) -> Result<()> {
+        let request = tonic::Request::new(ClearCacheRequest { id: batch_id }).inject_context();
+        self.stub.clear_cache(request).await?;
+        Ok(())
+    }
+
+    /// Filter a cached batch
+    #[instrument(skip(self))]
+    pub async fn filter_batch(
+        &mut self,
+        batch_id: u64,
+        request_ids: Vec<u64>,
+    ) -> Result<Option<CachedBatch>> {
+        let request = tonic::Request::new(FilterBatchRequest {
+            batch_id,
+            request_ids,
+        })
+        .inject_context();
+        let filtered_batch = self.stub.filter_batch(request).await?.into_inner();
+        Ok(filtered_batch.batch)
+    }
+
+    /// Warmup on a max size batch
+    ///
+    /// Returns the maximum amount of tokens supported by the hardware
+    #[instrument(skip_all)]
+    pub async fn warmup(
+        &mut self,
+        max_input_length: u32,
+        max_prefill_tokens: u32,
+        max_total_tokens: u32,
+        max_batch_total_tokens: u32,
+        max_batch_size: Option<usize>,
+    ) -> Result<Option<u32>> {
+        let mut n_tokens = 0;
+        let mut requests = Vec::new();
+        // Create requests
+        while n_tokens < max_prefill_tokens {
+            let truncate = min(max_input_length, max_prefill_tokens - n_tokens);
+
+            let mut input_chunks = Vec::new();
+            input_chunks
+                .push(Chunk::Text("_test ".to_string().repeat(max_input_length as usize)).into());
+            if n_tokens == 0 {
+                input_chunks.push(
+                    Chunk::Image(Image {
+                        // Safe unwrap, because we control the data.
+                        data: STANDARD.decode(WARMUP_IMAGE_BASE64).unwrap(),
+                        mimetype: "image/jpeg;base64".to_string(),
+                    })
+                    .into(),
+                );
+            }
+
+            // Send stringly-typed inputs for compatibility for backends that haven't
+            // been updated to support chunks.
+
+            let mut inputs = String::new();
+            inputs.push_str(&"_test ".to_string().repeat(max_input_length as usize));
+            if n_tokens == 0 {
+                // 1 request is enough to test vision heads.
+                // Sending images on other queries messes up easily with truncation.
+                inputs.push_str(&format!(
+                    "![](data:image/jpeg;base64,{WARMUP_IMAGE_BASE64})",
+                ));
+            }
+
+            requests.push(Request {
+                id: 0,
+                inputs,
+                input_chunks: Some(Input {
+                    chunks: input_chunks,
+                }),
+                // We truncate the input on the server side to be sure that it has the correct size
+                truncate,
+                // Most request will have that
+                add_special_tokens: true,
+                // Blocks and slots will be set on the server side if we use paged attention
+                blocks: vec![],
+                slots: vec![],
+                prefix_len: 0,
+                // Set sampling parameters to also take these ops into account in the max memory
+                parameters: Some(NextTokenChooserParameters {
+                    temperature: 0.9,
+                    top_k: 10,
+                    top_p: 0.9,
+                    typical_p: 0.9,
+                    do_sample: false,
+                    seed: 0,
+                    repetition_penalty: 1.2,
+                    frequency_penalty: 0.1,
+                    watermark: true,
+                    grammar: String::new(),
+                    grammar_type: GrammarType::None as i32,
+                }),
+                stopping_parameters: Some(StoppingCriteriaParameters {
+                    max_new_tokens: max_total_tokens - truncate,
+                    stop_sequences: vec![],
+                    ignore_eos_token: true,
+                }),
+                prefill_logprobs: true,
+                top_n_tokens: 20,
+                adapter_id: None,
+            });
+            n_tokens += max_input_length;
+
+            // Check max_batch_size
+            if Some(requests.len()) == max_batch_size {
+                break;
+            }
+        }
+
+        let batch = Batch {
+            id: 0,
+            size: requests.len() as u32,
+            requests,
+            max_tokens: max_input_length,
+            max_blocks: 0,
+        };
+
+        let request = tonic::Request::new(WarmupRequest {
+            batch: Some(batch),
+            max_input_length,
+            max_prefill_tokens,
+            max_total_tokens,
+            max_batch_total_tokens,
+        })
+        .inject_context();
+        let response = self.stub.warmup(request).await?.into_inner();
+        Ok(response.max_supported_total_tokens)
+    }
+
+    /// Generate one token for each request in the given batch
+    ///
+    /// Returns Generation for each request in batch
+    /// and the next cached batch
+    #[instrument(skip_all, fields(id = &batch.id, size = &batch.size))]
+    pub async fn prefill(
+        &mut self,
+        batch: Batch,
+    ) -> Result<(Vec<Generation>, Option<CachedBatch>, PrefillTimings)> {
+        let request = tonic::Request::new(PrefillRequest { batch: Some(batch) }).inject_context();
+        let response = self.stub.prefill(request).await?.into_inner();
+        Ok((
+            response.generations,
+            response.batch,
+            PrefillTimings::new(response.forward_ns, response.decode_ns, response.total_ns),
+        ))
+    }
+
+    /// Generate one token for each request in the given cached batches
+    ///
+    /// Returns Generation for each request in batches
+    /// and the next cached batch
+    #[instrument(skip_all, fields(size = batches.iter().map(|batch|{batch.size}).sum::<u32>()))]
+    pub async fn decode(
+        &mut self,
+        batches: Vec<CachedBatch>,
+    ) -> Result<(Vec<Generation>, Option<CachedBatch>, DecodeTimings)> {
+        let request = tonic::Request::new(DecodeRequest { batches }).inject_context();
+        let response = self.stub.decode(request).await?.into_inner();
+        Ok((
+            response.generations,
+            response.batch,
+            DecodeTimings::new(
+                response.concat_ns,
+                response.forward_ns,
+                response.decode_ns,
+                response.total_ns,
+            ),
+        ))
+    }
+}
+
+pub struct PrefillTimings {
+    pub forward: Duration,
+    pub decode: Duration,
+    pub total: Duration,
+}
+
+impl PrefillTimings {
+    fn new(forward_ns: u64, decode_ns: u64, total_ns: u64) -> Self {
+        Self {
+            forward: Duration::from_nanos(forward_ns),
+            decode: Duration::from_nanos(decode_ns),
+            total: Duration::from_nanos(total_ns),
+        }
+    }
+}
+
+pub struct DecodeTimings {
+    pub concat: Option<Duration>,
+    pub forward: Duration,
+    pub decode: Duration,
+    pub total: Duration,
+}
+
+impl DecodeTimings {
+    fn new(concat_ns: Option<u64>, forward_ns: u64, decode_ns: u64, total_ns: u64) -> Self {
+        Self {
+            concat: concat_ns.map(Duration::from_nanos),
+            forward: Duration::from_nanos(forward_ns),
+            decode: Duration::from_nanos(decode_ns),
+            total: Duration::from_nanos(total_ns),
+        }
+    }
+}
diff --git a/backends/client/src/v3/mod.rs b/backends/client/src/v3/mod.rs
new file mode 100644
index 00000000..4a1296a2
--- /dev/null
+++ b/backends/client/src/v3/mod.rs
@@ -0,0 +1,13 @@
+#[allow(clippy::derive_partial_eq_without_eq)]
+mod pb;
+
+mod client;
+mod sharded_client;
+
+pub use client::Client;
+pub use pb::generate::v3::{
+    input_chunk::Chunk, Batch, CachedBatch, FinishReason, GeneratedText, Generation, GrammarType,
+    HealthResponse, Image, InfoResponse, Input, InputChunk, NextTokenChooserParameters, Request,
+    StoppingCriteriaParameters, Tokens,
+};
+pub use sharded_client::ShardedClient;
diff --git a/backends/client/src/v3/sharded_client.rs b/backends/client/src/v3/sharded_client.rs
new file mode 100644
index 00000000..9709dfcb
--- /dev/null
+++ b/backends/client/src/v3/sharded_client.rs
@@ -0,0 +1,263 @@
+/// Multi shard Client
+use crate::{v3, Health, ShardInfo};
+use crate::{ClientError, Result};
+
+use crate::v3::{Chunk, InfoResponse, Input};
+use async_trait::async_trait;
+use futures::future::join_all;
+use tonic::transport::Uri;
+use tracing::instrument;
+use v3::client::{DecodeTimings, PrefillTimings};
+use v3::{
+    Batch, CachedBatch, Client, Generation, GrammarType, HealthResponse,
+    NextTokenChooserParameters, Request, StoppingCriteriaParameters,
+};
+
+#[derive(Debug, Clone)]
+/// Text Generation Inference gRPC multi client
+pub struct ShardedClient {
+    clients: Vec<Client>,
+}
+
+impl ShardedClient {
+    fn new(clients: Vec<Client>) -> Self {
+        Self { clients }
+    }
+
+    /// Create a new ShardedClient from a master client. The master client will communicate with
+    /// the other shards and returns all uris/unix sockets with the `service_discovery` gRPC method.
+    async fn from_master_client(mut master_client: Client) -> Result<Self> {
+        // Get all uris/unix sockets from the master client
+        let uris = master_client.service_discovery().await?;
+        let futures = uris.into_iter().map(Client::connect_uds);
+        let clients: Result<Vec<Client>> = join_all(futures).await.into_iter().collect();
+        Ok(Self::new(clients?))
+    }
+
+    /// Returns a client connected to the given uri
+    pub async fn connect(uri: Uri) -> Result<Self> {
+        let master_client = Client::connect(uri).await?;
+        Self::from_master_client(master_client).await
+    }
+
+    /// Returns a client connected to the given unix socket
+    pub async fn connect_uds(path: String) -> Result<Self> {
+        let master_client = Client::connect_uds(path).await?;
+        Self::from_master_client(master_client).await
+    }
+
+    /// Get the model info
+    #[instrument(skip(self))]
+    pub async fn info(&mut self) -> Result<ShardInfo> {
+        let futures: Vec<_> = self
+            .clients
+            .iter_mut()
+            .map(|client| client.info())
+            .collect();
+        join_all(futures).await.pop().unwrap().map(ShardInfo::from)
+    }
+
+    /// GRPC health check
+    #[instrument(skip(self))]
+    pub async fn health(&mut self) -> Result<HealthResponse> {
+        let futures: Vec<_> = self
+            .clients
+            .iter_mut()
+            .map(|client| client.health())
+            .collect();
+        join_all(futures).await.pop().unwrap()
+    }
+
+    /// Clear the past generations cache
+    #[instrument(skip(self))]
+    pub async fn clear_cache(&mut self, batch_id: Option<u64>) -> Result<()> {
+        let futures: Vec<_> = self
+            .clients
+            .iter_mut()
+            .map(|client| client.clear_cache(batch_id))
+            .collect();
+        join_all(futures).await.into_iter().collect()
+    }
+
+    /// Filter a cached batch
+    #[instrument(skip(self))]
+    pub async fn filter_batch(
+        &mut self,
+        batch_id: u64,
+        request_ids: Vec<u64>,
+    ) -> Result<Option<CachedBatch>> {
+        let futures: Vec<_> = self
+            .clients
+            .iter_mut()
+            .map(|client| Box::pin(client.filter_batch(batch_id, request_ids.clone())))
+            .collect();
+        // all shards return the same message
+        join_all(futures).await.pop().unwrap()
+    }
+
+    /// Warmup on a max size batch
+    ///
+    /// Returns the maximum amount of tokens supported by the hardware
+    #[instrument(skip(self))]
+    pub async fn warmup(
+        &mut self,
+        max_input_length: u32,
+        max_prefill_tokens: u32,
+        max_total_tokens: u32,
+        max_batch_total_tokens: u32,
+        max_batch_size: Option<usize>,
+    ) -> Result<Option<u32>> {
+        let futures: Vec<_> = self
+            .clients
+            .iter_mut()
+            .map(|client| {
+                Box::pin(client.warmup(
+                    max_input_length,
+                    max_prefill_tokens,
+                    max_total_tokens,
+                    max_batch_total_tokens,
+                    max_batch_size,
+                ))
+            })
+            .collect();
+        // Take the minimum value
+        let results = join_all(futures)
+            .await
+            .into_iter()
+            .collect::<Result<Vec<Option<u32>>>>()?;
+        Ok(results.into_iter().flatten().min())
+    }
+
+    /// Generate one token for each request in the given batch
+    ///
+    /// Returns Generation for each request in batch
+    /// and the next cached batch
+    #[instrument(skip_all, fields(id = & batch.id, size = & batch.size))]
+    pub async fn prefill(
+        &mut self,
+        batch: Batch,
+    ) -> Result<(Vec<Generation>, Option<CachedBatch>, PrefillTimings)> {
+        let futures: Vec<_> = self
+            .clients
+            .iter_mut()
+            .map(|client| Box::pin(client.prefill(batch.clone())))
+            .collect();
+        #[allow(clippy::type_complexity)]
+        let results: Result<Vec<(Vec<Generation>, Option<CachedBatch>, PrefillTimings)>> =
+            join_all(futures).await.into_iter().collect();
+        let mut results = results?;
+
+        let (mut generations, next_batch, mut timings) =
+            results.pop().ok_or(ClientError::EmptyResults)?;
+
+        // Merge generations from different model shards
+        for (mut shard_generations, _, shard_timings) in results.into_iter() {
+            generations.append(&mut shard_generations);
+            // Return the timings of the slowest shard
+            if shard_timings.total > timings.total {
+                timings = shard_timings;
+            }
+        }
+        Ok((generations, next_batch, timings))
+    }
+
+    /// Generate one token for each request in the given cached batches
+    ///
+    /// Returns Generation for each request in batches
+    /// and the next cached batch
+    #[instrument(skip_all, fields(size = batches.iter().map(| batch | {batch.size}).sum::< u32 > ()))]
+    pub async fn decode(
+        &mut self,
+        batches: Vec<CachedBatch>,
+    ) -> Result<(Vec<Generation>, Option<CachedBatch>, DecodeTimings)> {
+        let futures: Vec<_> = self
+            .clients
+            .iter_mut()
+            .map(|client| Box::pin(client.decode(batches.clone())))
+            .collect();
+        #[allow(clippy::type_complexity)]
+        let results: Result<Vec<(Vec<Generation>, Option<CachedBatch>, DecodeTimings)>> =
+            join_all(futures).await.into_iter().collect();
+        let mut results = results?;
+
+        let (mut generations, next_batch, mut timings) =
+            results.pop().ok_or(ClientError::EmptyResults)?;
+
+        // Merge generations from different model shards
+        for (mut shard_generations, _, shard_timings) in results.into_iter() {
+            generations.append(&mut shard_generations);
+            // Return the timings of the slowest shard
+            if shard_timings.total > timings.total {
+                timings = shard_timings;
+            }
+        }
+        Ok((generations, next_batch, timings))
+    }
+}
+
+impl From<InfoResponse> for ShardInfo {
+    fn from(value: InfoResponse) -> Self {
+        Self {
+            requires_padding: value.requires_padding,
+            dtype: value.dtype,
+            device_type: value.device_type,
+            window_size: value.window_size,
+            speculate: value.speculate,
+        }
+    }
+}
+
+#[async_trait]
+impl Health for ShardedClient {
+    async fn device_health(&self) -> Result<()> {
+        self.clone().health().await?;
+        Ok(())
+    }
+
+    async fn model_health(&self) -> Result<()> {
+        // Dummy batch of 1 token and 1 generated token
+        let liveness_request = Request {
+            id: u64::MAX,
+            inputs: "liveness".to_string(),
+            input_chunks: Some(Input {
+                chunks: vec![Chunk::Text("liveness".into()).into()],
+            }),
+            truncate: 10,
+            add_special_tokens: true,
+            prefill_logprobs: false,
+            parameters: Some(NextTokenChooserParameters {
+                temperature: 1.0,
+                top_k: 0,
+                top_p: 1.0,
+                typical_p: 1.0,
+                do_sample: false,
+                seed: 0,
+                repetition_penalty: 1.0,
+                frequency_penalty: 0.0,
+                watermark: false,
+                grammar: String::new(),
+                grammar_type: GrammarType::None as i32,
+            }),
+            stopping_parameters: Some(StoppingCriteriaParameters {
+                max_new_tokens: 1,
+                stop_sequences: vec![],
+                ignore_eos_token: false,
+            }),
+            top_n_tokens: 0,
+            // Block 0 is reserved for health checks
+            blocks: vec![0],
+            slots: (0..16).collect(),
+            prefix_len: 0,
+            adapter_id: None,
+        };
+        let batch = Batch {
+            id: u64::MAX,
+            requests: vec![liveness_request],
+            size: 1,
+            max_tokens: 2,
+            max_blocks: 1,
+        };
+        self.clone().prefill(batch).await?;
+        Ok(())
+    }
+}
diff --git a/router/grpc-metadata/Cargo.toml b/backends/grpc-metadata/Cargo.toml
similarity index 100%
rename from router/grpc-metadata/Cargo.toml
rename to backends/grpc-metadata/Cargo.toml
diff --git a/router/grpc-metadata/src/lib.rs b/backends/grpc-metadata/src/lib.rs
similarity index 100%
rename from router/grpc-metadata/src/lib.rs
rename to backends/grpc-metadata/src/lib.rs
diff --git a/backends/trtllm/CMakeLists.txt b/backends/trtllm/CMakeLists.txt
new file mode 100644
index 00000000..425b2d7b
--- /dev/null
+++ b/backends/trtllm/CMakeLists.txt
@@ -0,0 +1,63 @@
+cmake_minimum_required(VERSION 3.20)
+
+project(tgi-trtllm-backend VERSION 1.0.0)
+set(CMAKE_CXX_STANDARD 20)
+
+include(FetchContent)
+include(ExternalProject)
+
+option(TGI_TRTLLM_BACKEND_BUILD_TESTS "Enable building the unittests suite" OFF)
+option(TGI_TRTLLM_BACKEND_BUILD_EXAMPLES "Enable building the examples suite" OFF)
+set(TGI_TRTLLM_BACKEND_TARGET_CUDA_ARCH_LIST "89-real" CACHE STRING "List of CUDA architectures to support")
+set(TGI_TRTLLM_BACKEND_TRT_ROOT "/usr/local/tensorrt" CACHE STRING "Path where TensorRT libraries and headers are located")
+set(TGI_TRTLLM_BACKEND_TRT_INCLUDE_DIR "${TGI_TRTLLM_BACKEND_TRT_ROOT}/include" CACHE STRING "Path where TensorRT headers are located")
+set(TGI_TRTLLM_BACKEND_TRT_LIB_DIR "${TGI_TRTLLM_BACKEND_TRT_ROOT}/lib" CACHE STRING "Path where TensorRT libraries are located")
+
+# We are using nvidia-ml to query at runtime device information to enable some architecture-specific features
+find_package(CUDAToolkit 12.5 REQUIRED COMPONENTS CUDA::cudart CUDA::nvml)
+
+#### External dependencies ####
+include(cmake/fmt.cmake)
+include(cmake/json.cmake)
+include(cmake/spdlog.cmake)
+include(cmake/trtllm.cmake)
+
+# Let's build TRTLLM as part of CMake
+add_subdirectory("${trtllm_SOURCE_DIR}/cpp" "${trtllm_SOURCE_DIR}/..")
+
+# Tell CMake to need try to override the RPATH for executorWorker as it has not information on how to do so
+set_target_properties(executorWorker PROPERTIES SKIP_BUILD_RPATH TRUE)
+
+# TGI TRTLLM Backend definition
+add_library(tgi_trtllm_backend_impl STATIC include/backend.h lib/backend.cpp include/hardware.h)
+include_directories(${TGI_TRTLLM_BACKEND_TRT_INCLUDE_DIR})
+target_include_directories(tgi_trtllm_backend_impl PRIVATE
+        $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+        $<INSTALL_INTERFACE:include>
+)
+target_include_directories(tgi_trtllm_backend_impl PUBLIC "${trtllm_SOURCE_DIR}/cpp/include")
+target_link_libraries(tgi_trtllm_backend_impl PRIVATE tensorrt_llm nvinfer_plugin_tensorrt_llm tensorrt_llm_nvrtc_wrapper CUDA::cudart CUDA::nvml)
+target_link_libraries(tgi_trtllm_backend_impl PUBLIC nlohmann_json::nlohmann_json spdlog::spdlog fmt::fmt)
+
+# This install all the artifacts in CMAKE_INSTALL_PREFIX under include/ lib/ bin/ to make easy to link / find it back
+install(TARGETS tgi_trtllm_backend_impl tensorrt_llm nvinfer_plugin_tensorrt_llm decoder_attention executorWorker)
+install(FILES ${TRTLLM_NVRTC_WRAPPER_LIBRARY_PATH} ${TRTLLM_EXECUTOR_STATIC_LIBRARY_PATH} TYPE LIB)
+
+#### Unit Tests ####
+if (${TGI_TRTLLM_BACKEND_BUILD_TESTS})
+    message(STATUS "Building tests")
+    FetchContent_Declare(
+            Catch2
+            GIT_REPOSITORY https://github.com/catchorg/Catch2
+            GIT_TAG v3.6.0
+    )
+    FetchContent_MakeAvailable(Catch2)
+
+    #    add_executable(tgi_trtllm_backend_tests tests/infer_test.cpp)
+    #    target_link_libraries(tgi_trtllm_backend_tests PRIVATE tgi_trtllm_backend_impl Catch2::Catch2WithMain nlohmann_json::nlohmann_json spdlog::spdlog fmt::fmt CUDA::cudart CUDA::nvml)
+
+    list(APPEND CMAKE_MODULE_PATH ${catch2_SOURCE_DIR}/extras)
+    include(CTest)
+    include(Catch)
+    #    catch_discover_tests(tgi_trtllm_backend_tests)
+endif ()
diff --git a/backends/trtllm/Cargo.toml b/backends/trtllm/Cargo.toml
new file mode 100644
index 00000000..43a114ba
--- /dev/null
+++ b/backends/trtllm/Cargo.toml
@@ -0,0 +1,27 @@
+[package]
+name = "text-generation-backends-trtllm"
+version.workspace = true
+edition.workspace = true
+authors.workspace = true
+homepage.workspace = true
+
+[dependencies]
+async-trait = "0.1"
+async-stream = "0.3"
+clap = { version = "4.5", features = ["derive"] }
+cxx = "1.0"
+log = { version = "0.4", features = [] }
+text-generation-router = { path = "../../router" }
+tokenizers = { version = "0.19", features = ["hf-hub"] }
+tokio = { version = "1.38", features = ["rt", "rt-multi-thread", "parking_lot", "signal", "sync"] }
+tokio-stream = "0.1.15"
+thiserror = "1.0.62"
+tracing = "0.1"
+tracing-opentelemetry = "0.24"
+tracing-subscriber = { version = "0.3", features = ["json", "env-filter"] }
+parking_lot = "0.12"
+
+[build-dependencies]
+cmake = "0.1"
+cxx-build = { version = "1.0", features = ["parallel"] }
+pkg-config = "0.3"
diff --git a/backends/trtllm/Dockerfile b/backends/trtllm/Dockerfile
new file mode 100644
index 00000000..5fd2f89f
--- /dev/null
+++ b/backends/trtllm/Dockerfile
@@ -0,0 +1,101 @@
+ARG CUDA_ARCH_LIST="75-real;80-real;86-real;89-real;90-real"
+ARG OMPI_VERSION="4.1.6"
+
+# Build dependencies resolver stage
+FROM lukemathwalker/cargo-chef:latest AS chef
+WORKDIR /usr/src/text-generation-inference/backends/trtllm
+
+FROM chef AS planner
+COPY . .
+RUN cargo chef prepare --recipe-path recipe.json
+
+# CUDA dependent dependencies resolver stage
+FROM nvidia/cuda:12.5.1-cudnn-devel-ubuntu22.04 AS cuda-builder
+
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    apt update && apt install -y \
+    build-essential \
+    cmake \
+    curl \
+    gcc  \
+    g++ \
+    git \
+    git-lfs \
+    libssl-dev \
+    ninja-build \
+    pkg-config \
+    python3 \
+    python3-setuptools \
+    tar \
+    wget
+
+ENV TGI_INSTALL_PREFIX=/usr/local/tgi
+ENV TENSORRT_INSTALL_PREFIX=/usr/local/tensorrt
+
+# Install OpenMPI
+FROM cuda-builder AS mpi-builder
+ARG OMPI_VERSION
+
+ENV OMPI_TARBALL_FILENAME="openmpi-$OMPI_VERSION.tar.bz2"
+RUN wget "https://download.open-mpi.org/release/open-mpi/v4.1/$OMPI_TARBALL_FILENAME" -P /opt/src && \
+    mkdir /usr/src/mpi && \
+    tar -xf "/opt/src/$OMPI_TARBALL_FILENAME" -C /usr/src/mpi --strip-components=1 && \
+    cd /usr/src/mpi && \
+    ./configure --prefix=/usr/local/mpi --with-cuda=/usr/local/cuda && \
+    make -j all && \
+    make install && \
+    rm -rf "/opt/src/$OMPI_TARBALL_FILENAME"
+
+# Install TensorRT
+FROM cuda-builder AS trt-builder
+COPY backends/trtllm/scripts/install_tensorrt.sh /opt/install_tensorrt.sh
+RUN chmod +x /opt/install_tensorrt.sh && \
+    /opt/install_tensorrt.sh
+
+# Build Backend
+FROM cuda-builder AS tgi-builder
+WORKDIR /usr/src/text-generation-inference
+
+# Install Rust
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | bash -s -- -y && \
+    chmod -R a+w /root/.rustup && \
+    chmod -R a+w /root/.cargo
+
+ENV PATH="/root/.cargo/bin:$PATH"
+RUN cargo install cargo-chef
+
+# Cache dependencies
+COPY --from=planner /usr/src/text-generation-inference/backends/trtllm/recipe.json .
+RUN cargo chef cook --release --recipe-path recipe.json
+
+# Build actual TGI
+ARG CUDA_ARCH_LIST
+ENV CMAKE_PREFIX_PATH="/usr/local/mpi:/usr/local/tensorrt:$CMAKE_PREFIX_PATH"
+ENV LD_LIBRARY_PATH="/usr/local/mpi/lib:$LD_LIBRARY_PATH"
+ENV PKG_CONFIG_PATH="/usr/local/mpi/lib/pkgconfig:$PKG_CONFIG_PATH"
+
+COPY . .
+COPY --from=trt-builder /usr/local/tensorrt /usr/local/tensorrt
+COPY --from=mpi-builder /usr/local/mpi /usr/local/mpi
+RUN mkdir $TGI_INSTALL_PREFIX && mkdir "$TGI_INSTALL_PREFIX/include" && mkdir "$TGI_INSTALL_PREFIX/lib" && \
+    cd backends/trtllm && \
+    CMAKE_INSTALL_PREFIX=$TGI_INSTALL_PREFIX cargo build --release
+
+FROM nvidia/cuda:12.5.1-cudnn-runtime-ubuntu22.04 AS runtime
+WORKDIR /usr/local/tgi/bin
+
+ENV LD_LIBRARY_PATH="/usr/local/tgi/lib:/usr/local/tensorrt/lib:/usr/local/cuda/lib64/stubs:$LD_LIBRARY_PATH"
+
+COPY --from=mpi-builder /usr/local/mpi /usr/local/mpi
+COPY --from=trt-builder /usr/local/tensorrt /usr/local/tensorrt
+COPY --from=tgi-builder /usr/local/tgi /usr/local/tgi
+COPY --from=tgi-builder /usr/src/text-generation-inference/target/release/text-generation-backends-trtllm /usr/local/tgi/bin/text-generation-launcher
+
+FROM runtime
+
+LABEL co.huggingface.vendor="Hugging Face Inc."
+LABEL org.opencontainers.image.authors="hardware@hf.co"
+
+ENTRYPOINT ["./text-generation-launcher"]
+CMD ["--executor-worker", "/usr/local/tgi/bin/executorWorker"]
diff --git a/backends/trtllm/README.md b/backends/trtllm/README.md
new file mode 100644
index 00000000..94064504
--- /dev/null
+++ b/backends/trtllm/README.md
@@ -0,0 +1,46 @@
+# Text Generation Inference - TensorRT-LLM Backend Implementation
+
+## Description
+
+This folder provides the sources of the TensorRT-LLM backend implementation powered by TensorRT-LLM Executor new API
+
+## Simplified Request Sequence
+
+```mermaid
+sequenceDiagram
+    actor User
+    participant TextGenerationInference.HttpServer
+    participant TextGenerationInference.TensorRtLlmBackend
+    participant TextGenerationInference.TensorRtLlmWorkerThread
+    participant TensorRtLlm.Executor
+    participant Nvidia.Gpu
+    User ->> TextGenerationInference.HttpServer: POST /generate
+    TextGenerationInference.HttpServer ->> TextGenerationInference.TensorRtLlmBackend: Validate and forward inputs & parameters
+    TextGenerationInference.TensorRtLlmBackend ->> TextGenerationInference.TensorRtLlmWorkerThread: Allocate a new context and spawn a new thread to handle the request
+    TextGenerationInference.TensorRtLlmWorkerThread ->> TensorRtLlm.Executor: Submit the request to the In-Flight Batcher
+    activate Nvidia.Gpu
+    TensorRtLlm.Executor ->> Nvidia.Gpu: Add the request to the poll for execution
+    TensorRtLlm.Executor -->> TextGenerationInference.TensorRtLlmWorkerThread: Response with an unique request identifier
+    rect rgb(10, 92, 54)
+        loop every 100us
+            rect rgb(15, 81, 50)
+                alt Acquire lock to query executor
+                    TextGenerationInference.TensorRtLlmWorkerThread ->> TensorRtLlm.Executor: Poll request number of new token(s) generated
+                else There are new generated tokens
+                    TextGenerationInference.TensorRtLlmWorkerThread ->> TensorRtLlm.Executor: Retrieve newly generated tokens
+                    TensorRtLlm.Executor -->> TextGenerationInference.TensorRtLlmWorkerThread: Return decoded token information and potential error (omitted)
+                    rect rgb(11, 110, 79)
+                        alt Generated token is final
+                            TensorRtLlm.Executor ->> Nvidia.Gpu: Remove request from the scheduler and from the GPU
+                            TextGenerationInference.TensorRtLlmWorkerThread -->> User: Stream the remaining decoded tokens and flush the connection
+                        else Generated token is not final
+                            TextGenerationInference.TensorRtLlmWorkerThread -->> User: Stream token back to the user as they get decoded
+                        end
+                    end
+                end
+            end
+            deactivate Nvidia.Gpu
+        end
+    end
+
+```
diff --git a/backends/trtllm/build.rs b/backends/trtllm/build.rs
new file mode 100644
index 00000000..08638262
--- /dev/null
+++ b/backends/trtllm/build.rs
@@ -0,0 +1,150 @@
+use cxx_build::CFG;
+use pkg_config;
+use std::env;
+use std::env::consts::ARCH;
+use std::path::{absolute, PathBuf};
+
+const ADDITIONAL_BACKEND_LINK_LIBRARIES: [&str; 2] = ["spdlog", "fmt"];
+const CUDA_ARCH_LIST: Option<&str> = option_env!("CUDA_ARCH_LIST");
+const CUDA_REQUIRED_VERSION: &str = "12.5";
+const MPI_REQUIRED_VERSION: &str = "4.1";
+const INSTALL_PREFIX: Option<&str> = option_env!("CMAKE_INSTALL_PREFIX");
+const TENSORRT_ROOT_DIR: Option<&str> = option_env!("TENSORRT_ROOT_DIR");
+const NCCL_ROOT_DIR: Option<&str> = option_env!("NCCL_ROOT_DIR");
+
+// Dependencies
+const BACKEND_DEPS: [&str; 2] = ["tgi_trtllm_backend_impl", "tgi_trtllm_backend"];
+const CUDA_TRANSITIVE_DEPS: [&str; 4] = ["cuda", "cudart", "cublas", "nvidia-ml"];
+const TENSORRT_LLM_TRANSITIVE_DEPS: [(&str, &str); 5] = [
+    ("dylib", "tensorrt_llm"),
+    ("static", "tensorrt_llm_executor_static"),
+    ("dylib", "tensorrt_llm_nvrtc_wrapper"),
+    ("dylib", "nvinfer_plugin_tensorrt_llm"),
+    ("dylib", "decoder_attention"),
+];
+
+macro_rules! probe {
+    ($name: expr, $version: expr) => {
+        if let Err(_) = pkg_config::probe_library($name) {
+            pkg_config::probe_library(&format!("{}-{}", $name, $version))
+                .expect(&format!("Failed to locate {}", $name));
+        }
+    };
+}
+
+fn build_backend(is_debug: bool, opt_level: &str, out_dir: &PathBuf) -> (PathBuf, PathBuf) {
+    // Build the backend implementation through CMake
+    let install_path = INSTALL_PREFIX.unwrap_or("/usr/local/tgi");
+    let tensorrt_path = TENSORRT_ROOT_DIR.unwrap_or("/usr/local/tensorrt");
+    let cuda_arch_list = CUDA_ARCH_LIST.unwrap_or("90-real"); // Hopper by default
+
+    let mut install_path = PathBuf::from(install_path);
+    if !install_path.is_absolute() {
+        install_path = absolute(out_dir).expect("cannot happen").join(install_path);
+    }
+
+    let _ = cmake::Config::new(".")
+        .uses_cxx11()
+        .generator("Ninja")
+        .profile(match is_debug {
+            true => "Debug",
+            false => "Release",
+        })
+        .env("OPT_LEVEL", opt_level)
+        .define("CMAKE_INSTALL_PREFIX", &install_path)
+        .define("CMAKE_CUDA_COMPILER", "/usr/local/cuda/bin/nvcc")
+        .define("TGI_TRTLLM_BACKEND_TARGET_CUDA_ARCH_LIST", cuda_arch_list)
+        .define("TGI_TRTLLM_BACKEND_TRT_ROOT", tensorrt_path)
+        .build();
+
+    // Additional transitive CMake dependencies
+    let deps_folder = out_dir.join("build").join("_deps");
+    for dependency in ADDITIONAL_BACKEND_LINK_LIBRARIES {
+        let dep_name = match is_debug {
+            true => format!("{}d", dependency),
+            false => String::from(dependency),
+        };
+        let dep_path = deps_folder.join(format!("{}-build", dependency));
+        println!("cargo:rustc-link-search={}", dep_path.display());
+        println!("cargo:rustc-link-lib=static={}", dep_name);
+    }
+
+    // Emit linkage information from the artifacts we just built
+    let install_lib_path = install_path.join("lib");
+
+    println!(
+        r"cargo:warning=Adding link search path: {}",
+        install_lib_path.display()
+    );
+    println!(r"cargo:rustc-link-search={}", install_lib_path.display());
+
+    (PathBuf::from(install_path), deps_folder)
+}
+
+fn build_ffi_layer(deps_folder: &PathBuf) {
+    CFG.include_prefix = "backends/trtllm";
+    cxx_build::bridge("src/lib.rs")
+        .static_flag(true)
+        .include(deps_folder.join("fmt-src").join("include"))
+        .include(deps_folder.join("spdlog-src").join("include"))
+        .include(deps_folder.join("json-src").join("include"))
+        .include(deps_folder.join("trtllm-src").join("cpp").join("include"))
+        .include("/usr/local/cuda/include")
+        .include("/usr/local/tensorrt/include")
+        .file("src/ffi.cpp")
+        .std("c++20")
+        .compile("tgi_trtllm_backend");
+
+    println!("cargo:rerun-if-changed=CMakeLists.txt");
+    println!("cargo:rerun-if-changed=include/backend.h");
+    println!("cargo:rerun-if-changed=lib/backend.cpp");
+    println!("cargo:rerun-if-changed=include/ffi.h");
+    println!("cargo:rerun-if-changed=src/ffi.cpp");
+}
+
+fn main() {
+    // Misc variables
+    let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap());
+    let build_profile = env::var("PROFILE").unwrap();
+    let (is_debug, opt_level) = match build_profile.as_ref() {
+        "debug" => (true, "0"),
+        _ => (false, "3"),
+    };
+
+    // Build the backend
+    let (_backend_path, deps_folder) = build_backend(is_debug, opt_level, &out_dir);
+
+    // Build the FFI layer calling the backend above
+    build_ffi_layer(&deps_folder);
+
+    // Emit linkage search path
+    probe!("ompi", MPI_REQUIRED_VERSION);
+
+    // Probe CUDA & co. with pkg-config
+    CUDA_TRANSITIVE_DEPS.iter().for_each(|name| {
+        probe!(name, CUDA_REQUIRED_VERSION);
+    });
+
+    // NCCL is slightly trickier because it might not have a pkgconfig installed
+    let nccl_library_path_default = format!("/usr/local/{}-linux-gnu", ARCH);
+    let nccl_library_path = NCCL_ROOT_DIR.unwrap_or(&nccl_library_path_default);
+    println!(r"cargo:rustc-link-search=native={}", nccl_library_path);
+    println!("cargo:rustc-link-lib=dylib=nccl");
+
+    // TensorRT
+    let tensort_library_path = TENSORRT_ROOT_DIR.unwrap_or("/usr/local/tensorrt/lib");
+    println!(r"cargo:rustc-link-search=native={}", tensort_library_path);
+    println!("cargo:rustc-link-lib=dylib=nvinfer");
+
+    // TensorRT-LLM
+    TENSORRT_LLM_TRANSITIVE_DEPS
+        .iter()
+        .for_each(|(link_type, name)| {
+            println!("cargo:rustc-link-lib={}={}", link_type, name);
+        });
+
+    // Backend
+    BACKEND_DEPS.iter().for_each(|name| {
+        println!("cargo:rustc-link-lib=static={}", name);
+    });
+}
diff --git a/backends/trtllm/cmake/fmt.cmake b/backends/trtllm/cmake/fmt.cmake
new file mode 100644
index 00000000..f94a9c56
--- /dev/null
+++ b/backends/trtllm/cmake/fmt.cmake
@@ -0,0 +1,6 @@
+FetchContent_Declare(
+        fmt
+        GIT_REPOSITORY https://github.com/fmtlib/fmt
+        GIT_TAG 11.0.1
+)
+FetchContent_MakeAvailable(fmt)
diff --git a/backends/trtllm/cmake/json.cmake b/backends/trtllm/cmake/json.cmake
new file mode 100644
index 00000000..29e5753b
--- /dev/null
+++ b/backends/trtllm/cmake/json.cmake
@@ -0,0 +1,5 @@
+fetchcontent_declare(
+        json
+        URL https://github.com/nlohmann/json/releases/download/v3.11.3/json.tar.xz
+)
+fetchcontent_makeavailable(json)
diff --git a/backends/trtllm/cmake/spdlog.cmake b/backends/trtllm/cmake/spdlog.cmake
new file mode 100644
index 00000000..c4ee5c97
--- /dev/null
+++ b/backends/trtllm/cmake/spdlog.cmake
@@ -0,0 +1,17 @@
+set(SPDLOG_USE_FMT ON)
+set(SPDLOG_BUILD_SHARED OFF)
+set(SPDLOG_FMT_EXTERNAL ON)
+
+# Define the level at which SPDLOG_ compilation level is defined
+if (${CMAKE_BUILD_TYPE} STREQUAL "Debug")
+    add_compile_definitions(SPDLOG_ACTIVE_LEVEL SPDLOG_LEVEL_DEBUG)
+else ()
+    add_compile_definitions(SPDLOG_ACTIVE_LEVEL SPDLOG_LEVEL_INFO)
+endif ()
+
+fetchcontent_declare(
+        spdlog
+        GIT_REPOSITORY https://github.com/gabime/spdlog.git
+        GIT_TAG v1.14.1
+)
+fetchcontent_makeavailable(spdlog)
diff --git a/backends/trtllm/cmake/trtllm.cmake b/backends/trtllm/cmake/trtllm.cmake
new file mode 100644
index 00000000..e59ad4cf
--- /dev/null
+++ b/backends/trtllm/cmake/trtllm.cmake
@@ -0,0 +1,42 @@
+set(TRT_INCLUDE_DIR ${TGI_TRTLLM_BACKEND_TRT_INCLUDE_DIR})
+set(TRT_LIB_DIR ${TGI_TRTLLM_BACKEND_TRT_LIB_DIR})
+
+set(USE_CXX11_ABI ON)
+set(BUILD_PYT OFF)
+set(BUILD_PYBIND OFF)
+set(BUILD_MICRO_BENCHMARKS OFF)
+set(BUILD_BENCHMARKS OFF)
+set(BUILD_TESTS OFF)
+set(CMAKE_CUDA_ARCHITECTURES ${TGI_TRTLLM_BACKEND_TARGET_CUDA_ARCH_LIST})
+
+message(STATUS "Building for CUDA Architectures: ${CMAKE_CUDA_ARCHITECTURES}")
+
+if (${CMAKE_BUILD_TYPE} STREQUAL "Debug")
+    set(FAST_BUILD ON)
+    set(NVTX_DISABLE OFF)
+else ()
+    set(FAST_BUILD OFF)
+    set(FAST_MATH ON)
+    set(NVTX_DISABLE ON)
+endif ()
+
+fetchcontent_declare(
+        trtllm
+        GIT_REPOSITORY https://github.com/NVIDIA/TensorRT-LLM.git
+        GIT_TAG a681853d3803ee5893307e812530b5e7004bb6e1
+        GIT_SHALLOW FALSE
+)
+fetchcontent_makeavailable(trtllm)
+
+message(STATUS "Found TensorRT-LLM: ${trtllm_SOURCE_DIR}")
+execute_process(COMMAND git lfs install WORKING_DIRECTORY "${trtllm_SOURCE_DIR}/")
+execute_process(COMMAND git lfs pull WORKING_DIRECTORY "${trtllm_SOURCE_DIR}/")
+
+# TRTLLM use a JIT based *precompiled* library to generate some specific kernels, we are generating the path to this one here
+set(TRTLLM_NVRTC_LIBRARY_NAME "${CMAKE_SHARED_LIBRARY_PREFIX}tensorrt_llm_nvrtc_wrapper${CMAKE_SHARED_LIBRARY_SUFFIX}" CACHE INTERNAL "nvrtc wrapper library name")
+set(TRTLLM_NVRTC_WRAPPER_LIBRARY_PATH "${trtllm_SOURCE_DIR}/cpp/tensorrt_llm/kernels/decoderMaskedMultiheadAttention/decoderXQAImplJIT/nvrtcWrapper/${CMAKE_LIBRARY_ARCHITECTURE}/${TRTLLM_NVRTC_LIBRARY_NAME}"
+        CACHE INTERNAL "nvrtc wrapper library path")
+
+# The same Executor Static library
+set(TRTLLM_EXECUTOR_STATIC_LIBRARY_NAME "${CMAKE_SHARED_LIBRARY_PREFIX}tensorrt_llm_executor_static${CMAKE_STATIC_LIBRARY_SUFFIX}" CACHE INTERNAL "executor_static library name")
+set(TRTLLM_EXECUTOR_STATIC_LIBRARY_PATH "${trtllm_SOURCE_DIR}/cpp/tensorrt_llm/executor/${CMAKE_LIBRARY_ARCHITECTURE}/${TRTLLM_EXECUTOR_STATIC_LIBRARY_NAME}" CACHE INTERNAL "executor_static library path")
diff --git a/backends/trtllm/cmake/utils/detect_cuda_arch.cu b/backends/trtllm/cmake/utils/detect_cuda_arch.cu
new file mode 100644
index 00000000..e69de29b
diff --git a/backends/trtllm/include/backend.h b/backends/trtllm/include/backend.h
new file mode 100644
index 00000000..7990e76b
--- /dev/null
+++ b/backends/trtllm/include/backend.h
@@ -0,0 +1,121 @@
+//
+// Created by Morgan Funtowicz on 6/30/24.
+//
+
+#ifndef TGI_TRTLLM_BACKEND_H
+#define TGI_TRTLLM_BACKEND_H
+
+#include <cmath>
+#include <filesystem>
+#include <span>
+#include <vector>
+
+#include <nlohmann/json.hpp>
+
+#include <tensorrt_llm/runtime/common.h>
+#include <tensorrt_llm/executor/executor.h>
+#include <tensorrt_llm/plugins/api/tllmPlugin.h>
+
+using json = nlohmann::json;
+namespace tle = tensorrt_llm::executor;
+
+namespace huggingface::tgi::backends {
+    using RequestId = tle::IdType;
+    using TokenId = tle::TokenIdType;
+
+    /**
+     * Initialize all the components required by TRTLLM.
+     * It is required to call this function before attempting to load any engine
+     */
+    void InitializeBackend();
+
+    /**
+     *
+     * @param config TensorRT-LLM configuration object
+     * @param workerPath Path to the "executorWorker" provided by TensorRT-LLM when using orchestrator mode
+     * @return
+     */
+    tle::ExecutorConfig GetExecutorConfig(const json &config, const std::string &workerPath);
+
+    /**
+     * Get the sampling configuration from the parameters provided by TGI
+     * @param topK
+     * @param topP
+     * @param temperature
+     * @param repetition_penalty
+     * @param frequency_penalty
+     * @param seed
+     * @return
+     */
+    tle::SamplingConfig GetSamplingConfig(
+            uint32_t topK,
+            float_t topP,
+            float_t temperature,
+            float_t repetition_penalty,
+            float_t frequency_penalty,
+            uint64_t seed
+    );
+
+    /**
+     *
+     */
+    class TensorRtLlmBackend {
+    private:
+        const json config;
+        tle::Executor executor;
+
+    public:
+        explicit TensorRtLlmBackend(
+                const std::filesystem::path &engineFolder,
+                const std::filesystem::path &executorWorker
+        );
+
+        /**
+         * Indicate if the backend is ready to accept incoming request
+         * @return true if ready, false otherwise
+         */
+        [[nodiscard]] bool IsReady() const;
+
+        /**
+         * Query the executor for the number of token available for pulling
+         * @return
+         */
+        [[nodiscard]] size_t NumResponsesReady() const;
+
+        /**
+         * Submit a new generation task to the executor
+         * @param tokens
+         * @param topK
+         * @param topP
+         * @param temperature
+         * @param repetition_penalty
+         * @param frequency_penalty
+         * @param seed
+         * @return Request id related to this generation for reference
+         */
+        [[nodiscard]] RequestId Submit(
+                const std::vector<TokenId> &tokens,
+                int32_t topK,
+                float_t topP,
+                float_t temperature,
+                float_t repetition_penalty,
+                float_t frequency_penalty,
+                uint64_t seed
+        );
+
+        /**
+         *
+         * @param requestId The request id to poll the generation results
+         * @return
+         */
+        std::vector<tle::Response> Poll(RequestId requestId);
+
+        /**
+         * Stop the underlying executor
+         */
+        void Shutdown();
+    };
+}
+
+
+#endif //TGI_TRTLLM_BACKEND_H
diff --git a/backends/trtllm/include/ffi.h b/backends/trtllm/include/ffi.h
new file mode 100644
index 00000000..fe0be9fc
--- /dev/null
+++ b/backends/trtllm/include/ffi.h
@@ -0,0 +1,75 @@
+//
+// Created by mfuntowicz on 7/11/24.
+//
+
+#ifndef TGI_TRTLLM_BACKEND_FFI_H
+#define TGI_TRTLLM_BACKEND_FFI_H
+
+#include <cstddef>
+#include "backend.h"
+
+namespace huggingface::tgi::backends {
+    class TensorRtLlmBackendImpl;
+}
+
+#include "backends/trtllm/src/lib.rs.h"
+
+
+namespace huggingface::tgi::backends {
+
+//    struct GenerationContext;
+
+    class TensorRtLlmBackendImpl : public TensorRtLlmBackend {
+    public:
+        /***
+         *
+         * @param engineFolder
+         * @param executorWorker
+         */
+        TensorRtLlmBackendImpl(const std::string_view &engineFolder, const std::string_view &executorWorker);
+
+        /***
+         *
+         * @return
+         */
+        bool IsReady() const;
+
+        /***
+         *
+         * @param tokens
+         * @param topK
+         * @param topP
+         * @param temperature
+         * @param repetition_penalty
+         * @param frequency_penalty
+         * @param seed
+         * @return
+         */
+        [[nodiscard("returned request id should be used to refer to the request's generation result later on")]]
+        uint64_t
+        Submit(rust::Slice<const uint32_t> tokens, int32_t topK, float_t topP, float_t temperature,
+               float_t repetition_penalty, float_t frequency_penalty, uint64_t seed);
+
+        /***
+         *
+         * @param requestId
+         * @param ctx
+         * @param callback
+         * @return
+         */
+        size_t StreamTokens(
+                const RequestId requestId,
+                huggingface::tgi::backends::GenerationContext *ctx,
+                rust::Fn<void(huggingface::tgi::backends::GenerationContext *,
+                              huggingface::tgi::backends::GenerationStep)> callback);
+    };
+
+    /***
+    *
+    * @param engineFolder
+    * @return
+    */
+    std::unique_ptr<TensorRtLlmBackendImpl> CreateTensorRtLlmBackend(rust::Str engineFolder, rust::Str executorWorker);
+}
+
+#endif //TGI_TRTLLM_BACKEND_FFI_H
diff --git a/backends/trtllm/include/hardware.h b/backends/trtllm/include/hardware.h
new file mode 100644
index 00000000..da0bf4f3
--- /dev/null
+++ b/backends/trtllm/include/hardware.h
@@ -0,0 +1,59 @@
+//
+// Created by mfuntowicz on 7/23/24.
+//
+
+#ifndef TGI_TRTLLM_BACKEND_HARDWARE_H
+#define TGI_TRTLLM_BACKEND_HARDWARE_H
+
+#include <cstdint>
+#include <limits>
+#include <fmt/base.h>
+#include <spdlog/spdlog.h>
+#include <nvml.h>
+
+namespace huggingface::hardware::cuda {
+
+#define AMPERE_SM_MAJOR 8
+#define HOPPER_SM_MAJOR 8
+
+    /**
+     * Store information about the version of the CUDA Compute Capabilities detected on the device
+     */
+    struct CudaComputeCapabilities {
+        int32_t major;
+        int32_t minor;
+
+        [[nodiscard]] constexpr bool isPostAmpere() const { return major >= AMPERE_SM_MAJOR; }
+
+        [[nodiscard]] constexpr bool isPostHopper() const { return major >= HOPPER_SM_MAJOR; }
+    };
+
+    CudaComputeCapabilities GetCudaComputeCapabilities() {
+        // Get the compute capabilities of the current hardware
+        nvmlDevice_t device;
+        CudaComputeCapabilities capabilities{0, 0};
+        if (nvmlDeviceGetHandleByIndex_v2(0, &device) == NVML_SUCCESS) {
+            SPDLOG_DEBUG("Successfully acquired nvmlDevice_t = 0");
+            if (nvmlDeviceGetCudaComputeCapability(device, &capabilities.major, &capabilities.minor) == NVML_SUCCESS) {
+                SPDLOG_INFO("Detected sm_{:d}{:d} compute capabilities", capabilities.major, capabilities.minor);
+            }
+        }
+
+        return capabilities;
+    }
+
+    /**
+     * Return the number of GPU detected. If no GPU is detected, return size_t::max()
+     * @return
+     */
+    std::optional<size_t> GetNumDevices() {
+        uint32_t numGpus = 0;
+        if (nvmlDeviceGetCount_v2(&numGpus) == NVML_SUCCESS) {
+            return std::optional(numGpus);
+        } else {
+            return std::nullopt;
+        }
+    }
+}
+
+#endif //TGI_TRTLLM_BACKEND_HARDWARE_H
diff --git a/backends/trtllm/lib/backend.cpp b/backends/trtllm/lib/backend.cpp
new file mode 100644
index 00000000..c066a6d6
--- /dev/null
+++ b/backends/trtllm/lib/backend.cpp
@@ -0,0 +1,146 @@
+#include <fstream>
+
+#include <fmt/ranges.h>
+#include <spdlog/spdlog.h>
+#include <nvml.h>
+
+#include "backend.h"
+#include "hardware.h"
+
+void huggingface::tgi::backends::InitializeBackend() {
+    SPDLOG_INFO("Initializing Backend...");
+    nvmlInit_v2();
+    initTrtLlmPlugins();
+
+    const auto numGpus = huggingface::hardware::cuda::GetNumDevices();
+    if (numGpus.has_value()) {
+        SPDLOG_INFO("Detected {:d} Nvidia GPU(s)", numGpus.value());
+    } else {
+        SPDLOG_WARN("Failed to detected Nvidia GPU(s) on the system");
+    }
+}
+
+[[nodiscard]]
+tle::ExecutorConfig huggingface::tgi::backends::GetExecutorConfig(const json &config, const std::string &workerPath) {
+    tle::ExecutorConfig execConfig(1);
+
+    // Retrieve the compute capabilities to enable some options at runtime
+    const auto computeCapabilities = huggingface::hardware::cuda::GetCudaComputeCapabilities();
+
+    // Single engine (TP = PP = 1) -> using leader mode (no MPI involved)
+    if (config["/pretrained_config/mapping/world_size"_json_pointer].get<uint8_t>() == 1) {
+        SPDLOG_INFO("Detected single engine deployment, using leader mode");
+        execConfig.setParallelConfig(tle::ParallelConfig(
+                tle::CommunicationType::kMPI,
+                tle::CommunicationMode::kLEADER,
+                std::nullopt,
+                std::nullopt,
+                std::nullopt
+        ));
+    } else { // Multiple engines -> using orchestrator mode (MPI involved)
+        SPDLOG_INFO("Detected sharded engine deployment, using orchestrator mode");
+        execConfig.setParallelConfig(tle::ParallelConfig(
+                tle::CommunicationType::kMPI,
+                tle::CommunicationMode::kORCHESTRATOR,
+                std::nullopt,
+                std::nullopt,
+                tle::OrchestratorConfig(true, workerPath, nullptr, true)
+        ));
+    }
+
+    // Define some configuration variables
+    execConfig.setKvCacheConfig(tle::KvCacheConfig(true));
+    execConfig.setEnableChunkedContext(computeCapabilities.isPostAmpere());
+    return execConfig;
+}
+
+tle::SamplingConfig huggingface::tgi::backends::GetSamplingConfig(
+        uint32_t topK,
+        float_t topP,
+        float_t temperature,
+        float_t repetition_penalty,
+        float_t frequency_penalty,
+        uint64_t seed) {
+    return tle::SamplingConfig(
+            1,  // TGI only use a single beam
+            topK,
+            topP,
+            std::nullopt,
+            std::nullopt,
+            std::nullopt,
+            seed,
+            temperature,
+            temperature,
+            std::nullopt,
+            repetition_penalty,
+            std::nullopt,
+            frequency_penalty
+    );
+}
+
+huggingface::tgi::backends::TensorRtLlmBackend::TensorRtLlmBackend(
+        const std::filesystem::path &enginesFolder,
+        const std::filesystem::path &executorWorker
+) :
+        config(json::parse(std::ifstream(enginesFolder / "config.json"))),
+        executor(
+                enginesFolder,
+                tensorrt_llm::executor::ModelType::kDECODER_ONLY,
+                GetExecutorConfig(config, executorWorker.string()
+                )) {
+    SPDLOG_INFO(FMT_STRING("Engine (version={})"), config["/version"_json_pointer].get_ref<const std::string &>());
+}
+
+bool huggingface::tgi::backends::TensorRtLlmBackend::IsReady() const {
+    return executor.canEnqueueRequests();
+}
+
+[[nodiscard("Returned number of requests needs to be consumed")]]
+size_t huggingface::tgi::backends::TensorRtLlmBackend::NumResponsesReady() const {
+    return executor.getNumResponsesReady();
+}
+
+[[nodiscard("Returned request id needs to be provided back to gather generated tokens")]]
+tle::IdType huggingface::tgi::backends::TensorRtLlmBackend::Submit(
+        const std::vector<tle::TokenIdType> &tokens,
+        const int32_t topK,
+        const float_t topP,
+        const float_t temperature,
+        const float_t repetition_penalty,
+        const float_t frequency_penalty,
+        const uint64_t seed
+) {
+#ifdef NDEBUG
+    SPDLOG_DEBUG(
+            FMT_STRING("Submitting inference over {:d} tokens to the executor ({:d} already in-flight)"),
+            tokens.size(),
+            executor.getLatestIterationStats().back().numActiveRequests
+    );
+#else
+    SPDLOG_DEBUG(
+            FMT_STRING("Submitting inference [{}] to the executor ({:d} already in-flight)"),
+            fmt::join(tokens, ", "),
+            executor.getLatestIterationStats().front().numActiveRequests
+    );
+#endif
+
+    const auto maxNumTokens = config["/build_config/max_num_tokens"_json_pointer].get<size_t>();
+    const auto maxNewTokens = static_cast<int32_t>(std::max(1ul, maxNumTokens - tokens.size()));
+
+    const auto sampling = GetSamplingConfig(topK, topP, temperature, repetition_penalty, frequency_penalty, seed);
+    const auto output = tle::OutputConfig(true, false, false, true, false);
+    return executor.enqueueRequest(
+            tle::Request{tokens, maxNewTokens, true, sampling, output});
+}
+
+[[nodiscard("Generated tokens result must be used")]]
+std::vector<tle::Response> huggingface::tgi::backends::TensorRtLlmBackend::Poll(const tle::IdType requestId) {
+    SPDLOG_DEBUG(FMT_STRING("Polling status for request {:d}"), requestId);
+    return executor.awaitResponses(requestId);
+}
+
+
+void huggingface::tgi::backends::TensorRtLlmBackend::Shutdown() {
+    SPDLOG_INFO("Shutting down executor");
+    executor.shutdown();
+}
diff --git a/backends/trtllm/scripts/install_tensorrt.sh b/backends/trtllm/scripts/install_tensorrt.sh
new file mode 100755
index 00000000..e0e2dd17
--- /dev/null
+++ b/backends/trtllm/scripts/install_tensorrt.sh
@@ -0,0 +1,111 @@
+#!/bin/bash
+
+set -ex
+
+TRT_VER="10.2.0.19"
+CUDA_VER="12.5"
+CUDNN_VER="9.2.1.18-1"
+NCCL_VER="2.22.3-1+cuda12.5"
+CUBLAS_VER="12.5.3.2-1"
+NVRTC_VER="12.5.82-1"
+
+for i in "$@"; do
+    case $i in
+        --TRT_VER=?*) TRT_VER="${i#*=}";;
+        --CUDA_VER=?*) CUDA_VER="${i#*=}";;
+        --CUDNN_VER=?*) CUDNN_VER="${i#*=}";;
+        --NCCL_VER=?*) NCCL_VER="${i#*=}";;
+        --CUBLAS_VER=?*) CUBLAS_VER="${i#*=}";;
+        *) ;;
+    esac
+    shift
+done
+
+NVCC_VERSION_OUTPUT=$(nvcc --version)
+if [[ $(echo $NVCC_VERSION_OUTPUT | grep -oP "\d+\.\d+" | head -n 1) != ${CUDA_VER} ]]; then
+  echo "The version of pre-installed CUDA is not equal to ${CUDA_VER}."
+  exit 1
+fi
+
+install_ubuntu_requirements() {
+    apt-get update && apt-get install -y --no-install-recommends gnupg2 curl ca-certificates
+    ARCH=$(uname -m)
+    if [ "$ARCH" = "amd64" ];then ARCH="x86_64";fi
+    if [ "$ARCH" = "aarch64" ];then ARCH="sbsa";fi
+    curl -fsSLO https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/${ARCH}/cuda-keyring_1.0-1_all.deb
+    dpkg -i cuda-keyring_1.0-1_all.deb
+
+    apt-get update
+    if [[ $(apt list --installed | grep libcudnn9) ]]; then
+      apt-get remove --purge -y --allow-change-held-packages libcudnn9*
+    fi
+    if [[ $(apt list --installed | grep libnccl) ]]; then
+      apt-get remove --purge -y --allow-change-held-packages libnccl*
+    fi
+    if [[ $(apt list --installed | grep libcublas) ]]; then
+      apt-get remove --purge -y --allow-change-held-packages libcublas*
+    fi
+    if [[ $(apt list --installed | grep cuda-nvrtc-dev) ]]; then
+      apt-get remove --purge -y --allow-change-held-packages cuda-nvrtc-dev*
+    fi
+    CUBLAS_CUDA_VERSION=$(echo $CUDA_VER | sed 's/\./-/g')
+    apt-get install -y --no-install-recommends libcudnn9-cuda-12=${CUDNN_VER} libcudnn9-dev-cuda-12=${CUDNN_VER}
+    apt-get install -y --no-install-recommends libnccl2=${NCCL_VER} libnccl-dev=${NCCL_VER}
+    apt-get install -y --no-install-recommends libcublas-${CUBLAS_CUDA_VERSION}=${CUBLAS_VER} libcublas-dev-${CUBLAS_CUDA_VERSION}=${CUBLAS_VER}
+    # NVRTC static library doesn't exist in NGC PyTorch container.
+    NVRTC_CUDA_VERSION=$(echo $CUDA_VER | sed 's/\./-/g')
+    apt-get install -y --no-install-recommends cuda-nvrtc-dev-${NVRTC_CUDA_VERSION}=${NVRTC_VER}
+    apt-get clean
+    rm -rf /var/lib/apt/lists/*
+}
+
+install_centos_requirements() {
+    CUBLAS_CUDA_VERSION=$(echo $CUDA_VER | sed 's/\./-/g')
+    yum -y update
+    yum -y install epel-release
+    yum remove -y libnccl* && yum -y install libnccl-${NCCL_VER} libnccl-devel-${NCCL_VER}
+    yum remove -y libcublas* && yum -y install libcublas-${CUBLAS_CUDA_VERSION}-${CUBLAS_VER} libcublas-devel-${CUBLAS_CUDA_VERSION}-${CUBLAS_VER}
+    yum clean all
+}
+
+install_tensorrt() {
+    #PY_VERSION=$(python3 -c 'import sys; print(".".join(map(str, sys.version_info[0:2])))')
+    #PARSED_PY_VERSION=$(echo "${PY_VERSION//./}")
+    TRT_CUDA_VERSION="12.5"
+
+    if [ -z "$RELEASE_URL_TRT" ];then
+        ARCH=${TRT_TARGETARCH}
+        if [ -z "$ARCH" ];then ARCH=$(uname -m);fi
+        if [ "$ARCH" = "arm64" ];then ARCH="aarch64";fi
+        if [ "$ARCH" = "amd64" ];then ARCH="x86_64";fi
+        if [ "$ARCH" = "x86_64" ];then DIR_NAME="x64-agnostic"; else DIR_NAME=${ARCH};fi
+        if [ "$ARCH" = "aarch64" ];then OS1="Ubuntu22_04" && OS2="Ubuntu-22.04" && OS="ubuntu-22.04"; else OS1="Linux" && OS2="Linux" && OS="linux";fi
+        RELEASE_URL_TRT=https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.2.0/tars/TensorRT-${TRT_VER}.${OS2}.${ARCH}-gnu.cuda-${TRT_CUDA_VERSION}.tar.gz
+    fi
+    wget --no-verbose ${RELEASE_URL_TRT} -O /tmp/TensorRT.tar
+    tar -xf /tmp/TensorRT.tar -C /usr/local/
+    mv /usr/local/TensorRT-${TRT_VER} /usr/local/tensorrt
+    # pip3 install /usr/local/tensorrt/python/tensorrt-*-cp${PARSED_PY_VERSION}-*.whl
+    rm -rf /tmp/TensorRT.tar
+}
+
+# Install base packages depending on the base OS
+ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
+case "$ID" in
+  debian)
+    install_ubuntu_requirements
+    install_tensorrt
+    ;;
+  ubuntu)
+    install_ubuntu_requirements
+    install_tensorrt
+    ;;
+  centos)
+    install_centos_requirements
+    install_tensorrt
+    ;;
+  *)
+    echo "Unable to determine OS..."
+    exit 1
+    ;;
+esac
diff --git a/backends/trtllm/src/backend.rs b/backends/trtllm/src/backend.rs
new file mode 100644
index 00000000..b23aa6c0
--- /dev/null
+++ b/backends/trtllm/src/backend.rs
@@ -0,0 +1,330 @@
+use std::future::Future;
+use std::path::Path;
+use std::pin::{pin, Pin};
+use std::str::FromStr;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::{Arc, OnceLock};
+use std::task::{Context, Poll};
+use std::time::Duration;
+
+use async_trait::async_trait;
+use cxx::UniquePtr;
+use log::{error, warn};
+use tokenizers::Tokenizer;
+use tokio::sync::mpsc::{unbounded_channel, UnboundedSender};
+use tokio::time::{sleep, Instant};
+use tokio_stream::wrappers::UnboundedReceiverStream;
+use tokio_stream::{Stream, StreamExt};
+use tracing::{instrument, span, Level};
+
+// use tokio::sync::RwLock;
+use parking_lot::RwLock;
+use text_generation_router::infer::{Backend, GeneratedText, InferError, InferStreamResponse};
+use text_generation_router::validation::ValidationError::UnsupportedModality;
+use text_generation_router::validation::{Chunk, ValidGenerateRequest, ValidationError};
+use text_generation_router::{FinishReason, Token};
+
+use crate::errors::TensorRtLlmBackendError;
+use crate::ffi::{create_tensorrt_llm_backend, GenerationStep, TensorRtLlmBackendImpl};
+
+// Value used to poll the state of the generation stream
+static POLLING_INTERVAL_US: OnceLock<u64> = OnceLock::new();
+
+type InferResult<T> = Result<T, InferError>;
+
+pub(crate) struct Generation {
+    executor: Arc<RwLock<UniquePtr<TensorRtLlmBackendImpl>>>,
+    done: Arc<AtomicBool>,
+}
+
+/// Holds the user provided input to be executed along with a channel allowing
+/// to bubble up all the generated tokens for that tokens the to end stream.
+pub struct GenerationContext {
+    sender: UnboundedSender<InferResult<InferStreamResponse>>,
+    tokenizer: Arc<Tokenizer>,
+    tokens: Vec<u32>,
+    done: Arc<AtomicBool>,
+    queued: Instant,
+    start: Option<Instant>,
+}
+
+impl Stream for Generation {
+    type Item = usize;
+
+    fn poll_next(self: Pin<&mut Self>, ctx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
+        let interval = POLLING_INTERVAL_US.get_or_init(|| {
+            u64::from_str(option_env!("TRTLLM_BACKEND_POLLING_INTERVAL_US").unwrap_or("100"))
+                .expect("Invalid value provided for envvar POLLING_INTERVAL_US")
+        });
+
+        if !self.done.load(Ordering::Relaxed) {
+            let backend = pin!(self.executor.read());
+            let status = match backend.poll(ctx) {
+                Poll::Ready(executor_r) => {
+                    let ready = executor_r.num_responses_ready();
+                    if ready == 0 {
+                        Poll::Pending
+                    } else {
+                        Poll::Ready(Some(ready))
+                    }
+                }
+                Poll::Pending => Poll::Pending,
+            };
+
+            let waker = ctx.waker().clone();
+            tokio::spawn(async {
+                sleep(Duration::from_micros(*interval)).await;
+                waker.wake();
+            });
+
+            status
+        } else {
+            Poll::Ready(None) // end of stream
+        }
+    }
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        (1, None)
+    }
+}
+
+unsafe impl Send for TensorRtLlmBackendImpl {}
+unsafe impl Sync for TensorRtLlmBackendImpl {}
+
+/// Implements the logic to execute generation with TensorRT-LLM executor API in background
+pub struct TensorRtLlmBackend {
+    tokenizer: Arc<Tokenizer>,
+
+    // Backing the backend behind a RwLock to allow concurrent read access to retrieve
+    // the number of available tokens (read only) in the Generation stream
+    backend: Arc<RwLock<UniquePtr<TensorRtLlmBackendImpl>>>,
+}
+
+impl TensorRtLlmBackend {
+    pub fn new<P: AsRef<Path> + Send + 'static, PP: AsRef<Path> + Send + 'static>(
+        tokenizer: Tokenizer,
+        engine_folder: P,
+        executor_worker_path: PP,
+    ) -> Result<Self, TensorRtLlmBackendError> {
+        Ok(TensorRtLlmBackend {
+            tokenizer: Arc::new(tokenizer),
+            backend: Arc::new(RwLock::new(create_tensorrt_llm_backend(
+                engine_folder.as_ref().to_str().unwrap(),
+                executor_worker_path.as_ref().to_str().unwrap(),
+            ))),
+        })
+    }
+
+    fn validate(request: &ValidGenerateRequest) -> InferResult<&String> {
+        if request.top_n_tokens > 1 {
+            return Err(InferError::ValidationError(
+                ValidationError::TopNTokensDisabled,
+            ));
+        }
+
+        // TODO: Is it really needed? How can it be validated before?
+        if request.parameters.grammar.is_some() {
+            return Err(InferError::ValidationError(ValidationError::Grammar));
+        }
+
+        match request.inputs.len() {
+            0 => Err(InferError::ValidationError(ValidationError::EmptyInput)),
+            2.. => Err(InferError::GenerationError(
+                "TensorRT-LLM backend don't support multi-chunk".into(),
+            )),
+            1 => match request.inputs.first().expect("Single item-chunk") {
+                Chunk::Text(text) => Ok(text),
+                Chunk::Image(_) => Err(InferError::ValidationError(UnsupportedModality("image"))),
+            },
+        }
+    }
+
+    fn generate(
+        &self,
+        sender: UnboundedSender<InferResult<InferStreamResponse>>,
+        tokens: Vec<u32>,
+        top_k: u32,
+        top_p: f32,
+        temperature: f32,
+        repetition_penalty: f32,
+        frequency_penalty: f32,
+        seed: u64,
+    ) {
+        let tokenizer = Arc::clone(&self.tokenizer);
+        let executor = Arc::clone(&self.backend);
+
+        // Let's push this in async context
+        tokio::spawn(async move {
+            // Define the generation state
+            let mut generation = Generation {
+                executor: executor.clone(),
+                done: Arc::new(AtomicBool::new(false)),
+            };
+
+            // Define the context over the generation
+            // TODO(asap): Do we really need so many shared-ownership?
+            let ctx = Box::new(GenerationContext {
+                sender: sender.clone(),
+                tokenizer,
+                tokens: vec![],
+                done: Arc::clone(&generation.done),
+                start: None,
+                queued: Instant::now(),
+            });
+
+            // We are leaking the context on-purpose to avoid the box being dropped while there are
+            // still computation ongoing
+            // TODO(asap): Can we achieve the same with an Arc<Box<T>> without the need to go unsafe?
+            let ctx_ = Box::leak(ctx);
+
+            // Submit the request to the batcher
+            let request_id = span!(Level::DEBUG, "submit")
+                .in_scope(|| async {
+                    let mut handle = executor.write().await;
+                    let request_id = handle.pin_mut().submit(
+                        &tokens,
+                        top_k as i32,
+                        top_p,
+                        temperature,
+                        repetition_penalty,
+                        frequency_penalty,
+                        seed,
+                    );
+
+                    request_id
+                })
+                .await;
+
+            while let Some(_) = generation.next().await {
+                let mut executor_w = executor.write().await;
+                let executor = executor_w.pin_mut();
+
+                span!(Level::DEBUG, "decode")
+                    .in_scope(|| async {
+                        unsafe {
+                            executor.stream_tokens(
+                                request_id,
+                                ctx_,
+                                |ctx: *mut GenerationContext, step: GenerationStep| {
+                                    let inner_ctx = &mut *ctx;
+
+                                    // Update the timestamp at which the request started effectively
+                                    // Can be a bit off, would need to be before the callback, let's see
+                                    inner_ctx.start.get_or_insert(Instant::now());
+                                    inner_ctx.done.store(step.is_final, Ordering::Relaxed);
+
+                                    // Ensure we are not running into errors
+                                    let parcel = if !step.has_error {
+                                        // Insert the latest generated token to the tracker
+                                        inner_ctx.tokens.push(step.token_id);
+
+                                        // Decode the token
+                                        let text = inner_ctx
+                                            .tokenizer
+                                            .decode(&[step.token_id], true)
+                                            .expect("Failed to decode token");
+
+                                        let special = inner_ctx
+                                            .tokenizer
+                                            .get_added_vocabulary()
+                                            .is_special_token(&text);
+
+                                        // Create the structure holding the token
+                                        let token = Token {
+                                            id: step.token_id,
+                                            text,
+                                            logprob: step.log_prob,
+                                            special,
+                                        };
+
+                                        if step.is_final {
+                                            let generated_text = inner_ctx
+                                                .tokenizer
+                                                .decode(&inner_ctx.tokens, true)
+                                                .expect("Failed to decode generated_tokens");
+
+                                            Ok(InferStreamResponse::End {
+                                                token,
+                                                top_tokens: vec![],
+                                                generated_text: GeneratedText {
+                                                    text: generated_text,
+                                                    generated_tokens: inner_ctx.tokens.len() as u32,
+                                                    finish_reason: FinishReason::EndOfSequenceToken,
+                                                    seed: None,
+                                                },
+                                                start: inner_ctx.start.unwrap_or(Instant::now()),
+                                                queued: inner_ctx.queued,
+                                            })
+                                        } else {
+                                            Ok(InferStreamResponse::Intermediate {
+                                                token,
+                                                top_tokens: vec![],
+                                            })
+                                        }
+                                    } else {
+                                        error!("Error caught while decoding: {}", &step.error_msg);
+                                        Err(InferError::GenerationError(step.error_msg))
+                                    };
+
+                                    // Send the parcel to the client
+                                    inner_ctx
+                                        .sender
+                                        .send(parcel)
+                                        .expect("Failed to sent msg through the channel");
+                                },
+                            );
+                        }
+                    })
+                    .await;
+            }
+
+            // "Properly" free the shared context...
+            // TODO: clean that piece of sh** asap
+            unsafe {
+                let _ = Box::from_raw(ctx_);
+            }
+        });
+    }
+}
+
+#[async_trait]
+impl Backend for TensorRtLlmBackend {
+    #[instrument(skip_all)]
+    fn schedule(
+        &self,
+        request: ValidGenerateRequest,
+    ) -> InferResult<UnboundedReceiverStream<InferResult<InferStreamResponse>>> {
+        // Let's add a few more validation
+        let input = TensorRtLlmBackend::validate(&request)?;
+
+        // Channel to stream the generated token as they come from the worker thread back to the transport layer
+        let (sender, receiver) = unbounded_channel();
+
+        // Unpack parameters
+        let params = &request.parameters;
+
+        // Preprocess the inputs to send to TRTLLM backend
+        let encoding = self
+            .tokenizer
+            .encode(input.as_str(), true)
+            .map_err(|e| InferError::GenerationError(e.to_string()))?;
+
+        // Generate the response
+        self.generate(
+            sender,
+            Vec::from(encoding.get_ids()),
+            params.top_k,
+            params.top_p,
+            params.temperature,
+            params.repetition_penalty,
+            params.frequency_penalty,
+            params.seed,
+        );
+
+        Ok(UnboundedReceiverStream::new(receiver))
+    }
+
+    async fn health(&self, _current_health: bool) -> bool {
+        true
+    }
+}
diff --git a/backends/trtllm/src/errors.rs b/backends/trtllm/src/errors.rs
new file mode 100644
index 00000000..a672d2a4
--- /dev/null
+++ b/backends/trtllm/src/errors.rs
@@ -0,0 +1,15 @@
+use thiserror::Error;
+
+use text_generation_router::server;
+
+#[derive(Debug, Error)]
+pub enum TensorRtLlmBackendError {
+    #[error("Tokenizer error: {0}")]
+    Tokenizer(String),
+    #[error("Argument validation error: {0}")]
+    ArgumentValidation(String),
+    #[error("WebServer error: {0}")]
+    WebServer(#[from] server::WebServerError),
+    #[error("Tokio runtime failed to start: {0}")]
+    Tokio(#[from] std::io::Error),
+}
diff --git a/backends/trtllm/src/ffi.cpp b/backends/trtllm/src/ffi.cpp
new file mode 100644
index 00000000..d6317a68
--- /dev/null
+++ b/backends/trtllm/src/ffi.cpp
@@ -0,0 +1,84 @@
+//
+// Created by mfuntowicz on 6/30/24.
+//
+#pragma once
+
+#include <cmath>
+#include <exception>
+#include <filesystem>
+#include <limits>
+#include <iterator>
+#include <vector>
+
+#include <spdlog/spdlog.h>
+#include "backends/trtllm/include/ffi.h"
+
+
+huggingface::tgi::backends::TensorRtLlmBackendImpl::TensorRtLlmBackendImpl(
+        const std::string_view &engineFolder,
+        const std::string_view &executorWorker
+) : TensorRtLlmBackend(engineFolder, executorWorker) {}
+
+
+bool huggingface::tgi::backends::TensorRtLlmBackendImpl::IsReady() const {
+    return TensorRtLlmBackend::IsReady();
+}
+
+uint64_t huggingface::tgi::backends::TensorRtLlmBackendImpl::Submit(
+        rust::Slice<const uint32_t> tokens, int32_t topK, float_t topP, float_t temperature, float_t repetition_penalty,
+        float_t frequency_penalty, uint64_t seed) {
+
+    // This will copy all the items from the initial slice
+    std::vector<int32_t> tokens_(std::make_move_iterator(tokens.begin()), std::make_move_iterator(tokens.end()));
+    return TensorRtLlmBackend::Submit(
+            std::move(tokens_), topK, topP, temperature, repetition_penalty, frequency_penalty, seed);
+}
+
+size_t huggingface::tgi::backends::TensorRtLlmBackendImpl::StreamTokens(
+        const uint64_t requestId,
+        huggingface::tgi::backends::GenerationContext *ctx,
+        rust::Fn<void(huggingface::tgi::backends::GenerationContext *,
+                      huggingface::tgi::backends::GenerationStep)> callback) {
+
+    size_t numTokens = 0;
+    for (const auto &item: Poll(requestId)) {
+        GenerationStep step;
+        if (!item.hasError()) {
+            SPDLOG_DEBUG("\tStreamTokens -> Decoding token...");
+            const auto decoded = item.getResult();
+
+            const auto token = decoded.outputTokenIds[0][0];
+            const auto isFinal = decoded.isFinal;
+            const auto logProb = decoded.logProbs.value()[0][0];
+
+            ++numTokens;
+
+            SPDLOG_DEBUG(FMT_STRING("\tStreamTokens -> {:d} {:.2f} (final = {})"), token, logProb, isFinal);
+            step = huggingface::tgi::backends::GenerationStep{
+                    static_cast<uint32_t>(token), logProb, isFinal, false, std::move(std::string())
+            };
+            SPDLOG_DEBUG("\tStreamTokens -> Post callback");
+        } else {
+            // TODO : Return rest::Result with error
+            const auto what = item.getErrorMsg();
+            SPDLOG_WARN("\tStreamTokens -> Got error while decoding: {}", what);
+            step = huggingface::tgi::backends::GenerationStep{
+                    std::numeric_limits<uint32_t>::max(), 0.0, true, true, std::move(what)
+            };
+        }
+
+        callback(std::move(ctx), std::move(step));
+    }
+
+    return numTokens;
+}
+
+std::unique_ptr<huggingface::tgi::backends::TensorRtLlmBackendImpl>
+huggingface::tgi::backends::CreateTensorRtLlmBackend(rust::Str engineFolder, rust::Str executorWorker) {
+    // Unconditionally call this to initialize and discover TRTLLM plugins
+    InitializeBackend();
+
+    const auto enginePath = std::string_view(engineFolder.begin(), engineFolder.end());
+    const auto executorPath = std::string_view(executorWorker.begin(), executorWorker.end());
+    return std::make_unique<TensorRtLlmBackendImpl>(std::move(enginePath), std::move(executorPath));
+}
diff --git a/backends/trtllm/src/lib.rs b/backends/trtllm/src/lib.rs
new file mode 100644
index 00000000..1a804f88
--- /dev/null
+++ b/backends/trtllm/src/lib.rs
@@ -0,0 +1,78 @@
+pub use backend::{GenerationContext, TensorRtLlmBackend};
+
+mod backend;
+pub mod errors;
+
+#[cxx::bridge(namespace = "huggingface::tgi::backends")]
+mod ffi {
+
+    /// Struct used as shared type between rust and C++ to represent the result
+    /// of a single decoding iteration
+    pub struct GenerationStep {
+        token_id: u32,
+        log_prob: f32,
+        is_final: bool,
+        has_error: bool,
+        error_msg: String,
+    }
+
+    extern "Rust" {
+        type GenerationContext;
+    }
+
+    unsafe extern "C++" {
+        include!("backends/trtllm/src/ffi.cpp");
+
+        /// Represent an instance of the underlying TensorRT-LLM backend
+        type TensorRtLlmBackendImpl;
+
+        /// Create an instance backed behind a std::unique_ptr to manage the lifespan of the backend
+        ///
+        /// # Arguments
+        ///
+        /// * `engine_folder`: Path to the folder containing all the TRTLLM engines
+        /// * `executor_worker`: Path to the TRTLLM executor worker
+        ///
+        /// returns: <unknown>
+        ///
+        /// # Examples
+        ///
+        /// ```
+        ///
+        /// ```
+        #[rust_name = "create_tensorrt_llm_backend"]
+        fn CreateTensorRtLlmBackend(
+            engine_folder: &str,
+            executor_worker: &str,
+        ) -> UniquePtr<TensorRtLlmBackendImpl>;
+
+        // #[rust_name = "is_ready"]
+        // fn IsReady(self: &TensorRtLlmBackendImpl) -> bool;
+
+        #[rust_name = "num_responses_ready"]
+        fn NumResponsesReady(self: &TensorRtLlmBackendImpl) -> usize;
+
+        #[rust_name = "submit"]
+        fn Submit(
+            self: Pin<&mut TensorRtLlmBackendImpl>,
+            tokens: &[u32],
+            top_k: i32,
+            top_p: f32,
+            temperature: f32,
+            repetition_penalty: f32,
+            frequency_penalty: f32,
+            seed: u64,
+        ) -> u64;
+
+        #[rust_name = "stream_tokens"]
+        unsafe fn StreamTokens(
+            self: Pin<&mut TensorRtLlmBackendImpl>,
+            request_id: u64,
+            ctx: *mut GenerationContext,
+            cb: unsafe fn(*mut GenerationContext, GenerationStep),
+        ) -> usize;
+
+        // #[rust_name = "shutdown"]
+        // fn Shutdown(self: Pin<&mut TensorRtLlmBackendImpl>);
+    }
+}
diff --git a/backends/trtllm/src/main.rs b/backends/trtllm/src/main.rs
new file mode 100644
index 00000000..e0ba46c7
--- /dev/null
+++ b/backends/trtllm/src/main.rs
@@ -0,0 +1,166 @@
+use clap::Parser;
+use std::collections::HashMap;
+use std::path::PathBuf;
+use text_generation_backends_trtllm::errors::TensorRtLlmBackendError;
+use text_generation_backends_trtllm::TensorRtLlmBackend;
+use text_generation_router::server;
+use tokenizers::{FromPretrainedParameters, Tokenizer};
+
+/// App Configuration
+#[derive(Parser, Debug)]
+#[clap(author, version, about, long_about = None)]
+struct Args {
+    #[clap(default_value = "128", long, env)]
+    max_concurrent_requests: usize,
+    #[clap(default_value = "2", long, env)]
+    max_best_of: usize,
+    #[clap(default_value = "4", long, env)]
+    max_stop_sequences: usize,
+    #[clap(default_value = "5", long, env)]
+    max_top_n_tokens: u32,
+    #[clap(default_value = "1024", long, env)]
+    max_input_tokens: usize,
+    #[clap(default_value = "2048", long, env)]
+    max_total_tokens: usize,
+    #[clap(default_value = "4096", long, env)]
+    max_batch_prefill_tokens: u32,
+    #[clap(long, env)]
+    max_batch_total_tokens: Option<u32>,
+    #[clap(default_value = "0.0.0.0", long, env)]
+    hostname: String,
+    #[clap(default_value = "3000", long, short, env)]
+    port: u16,
+    #[clap(long, env, required = true)]
+    tokenizer_name: String,
+    #[clap(long, env)]
+    tokenizer_config_path: Option<String>,
+    #[clap(long, env)]
+    revision: Option<String>,
+    #[clap(long, env)]
+    model_id: String,
+    #[clap(default_value = "2", long, env)]
+    validation_workers: usize,
+    #[clap(long, env)]
+    json_output: bool,
+    #[clap(long, env)]
+    otlp_endpoint: Option<String>,
+    #[clap(default_value = "text-generation-inference.router", long, env)]
+    otlp_service_name: String,
+    #[clap(long, env)]
+    cors_allow_origin: Option<Vec<String>>,
+    #[clap(long, env, default_value_t = false)]
+    messages_api_enabled: bool,
+    #[clap(default_value = "4", long, env)]
+    max_client_batch_size: usize,
+    #[clap(long, env)]
+    auth_token: Option<String>,
+    #[clap(long, env, help = "Path to the TensorRT-LLM Orchestrator worker")]
+    executor_worker: PathBuf,
+}
+
+#[tokio::main]
+async fn main() -> Result<(), TensorRtLlmBackendError> {
+    // Get args
+    let args = Args::parse();
+    // Pattern match configuration
+    let Args {
+        max_concurrent_requests,
+        max_best_of,
+        max_stop_sequences,
+        max_top_n_tokens,
+        max_input_tokens,
+        max_total_tokens,
+        max_batch_prefill_tokens,
+        max_batch_total_tokens,
+        hostname,
+        port,
+        tokenizer_name,
+        tokenizer_config_path,
+        revision,
+        model_id,
+        validation_workers,
+        json_output,
+        otlp_endpoint,
+        otlp_service_name,
+        cors_allow_origin,
+        messages_api_enabled,
+        max_client_batch_size,
+        auth_token,
+        executor_worker,
+    } = args;
+
+    // Launch Tokio runtime
+    text_generation_router::logging::init_logging(otlp_endpoint, otlp_service_name, json_output);
+
+    // Validate args
+    if max_input_tokens >= max_total_tokens {
+        return Err(TensorRtLlmBackendError::ArgumentValidation(
+            "`max_input_tokens` must be < `max_total_tokens`".to_string(),
+        ));
+    }
+    if max_input_tokens as u32 > max_batch_prefill_tokens {
+        return Err(TensorRtLlmBackendError::ArgumentValidation(format!("`max_batch_prefill_tokens` must be >= `max_input_tokens`. Given: {max_batch_prefill_tokens} and {max_input_tokens}")));
+    }
+
+    if validation_workers == 0 {
+        return Err(TensorRtLlmBackendError::ArgumentValidation(
+            "`validation_workers` must be > 0".to_string(),
+        ));
+    }
+
+    if let Some(ref max_batch_total_tokens) = max_batch_total_tokens {
+        if max_batch_prefill_tokens > *max_batch_total_tokens {
+            return Err(TensorRtLlmBackendError::ArgumentValidation(format!("`max_batch_prefill_tokens` must be <= `max_batch_total_tokens`. Given: {max_batch_prefill_tokens} and {max_batch_total_tokens}")));
+        }
+        if max_total_tokens as u32 > *max_batch_total_tokens {
+            return Err(TensorRtLlmBackendError::ArgumentValidation(format!("`max_total_tokens` must be <= `max_batch_total_tokens`. Given: {max_total_tokens} and {max_batch_total_tokens}")));
+        }
+    }
+
+    if !executor_worker.exists() {
+        return Err(TensorRtLlmBackendError::ArgumentValidation(format!(
+            "`executor_work` specified path doesn't exists: {}",
+            executor_worker.display()
+        )));
+    }
+
+    // Run server
+    let tokenizer = Tokenizer::from_pretrained(
+        tokenizer_name.clone(),
+        Some(FromPretrainedParameters {
+            revision: revision.clone().unwrap_or(String::from("main")),
+            user_agent: HashMap::new(),
+            auth_token,
+        }),
+    )
+    .map_err(|e| TensorRtLlmBackendError::Tokenizer(e.to_string()))?;
+
+    let backend = TensorRtLlmBackend::new(tokenizer, model_id, executor_worker)?;
+    server::run(
+        backend,
+        max_concurrent_requests,
+        max_best_of,
+        max_stop_sequences,
+        max_top_n_tokens,
+        max_input_tokens,
+        max_total_tokens,
+        validation_workers,
+        None,
+        tokenizer_name,
+        tokenizer_config_path,
+        revision,
+        hostname,
+        port,
+        cors_allow_origin,
+        false,
+        None,
+        None,
+        messages_api_enabled,
+        true,
+        max_client_batch_size,
+        false,
+        false,
+    )
+    .await?;
+    Ok(())
+}
diff --git a/backends/trtllm/tests/infer_test.cpp b/backends/trtllm/tests/infer_test.cpp
new file mode 100644
index 00000000..8520065a
--- /dev/null
+++ b/backends/trtllm/tests/infer_test.cpp
@@ -0,0 +1,14 @@
+//
+// Created by mfuntowicz on 7/2/24.
+//
+#include <catch2/catch_all.hpp>
+#include <spdlog/spdlog.h>
+#include "../include/backend.h"
+
+TEST_CASE("Load TRTLLM Engine on the TGI Backend", "[trtllm][engine][load]") {
+    const auto engines = std::filesystem::path("/home/mfuntowicz/.cache/huggingface/assets/trtllm/0.11.0.dev2024062500/meta-llama--Meta-Llama-3-8B-Instruct/4090/engines/");
+    const auto executor = std::filesystem::path("/home/mfuntowicz/Workspace/text-generation-inference/backends/trtllm/cmake-build-debug/cmake-build-debug/_deps/trtllm-src/cpp/tensorrt_llm/executor_worker/executorWorker");
+
+    spdlog::info("Loading config from: {}", absolute(engines).string());
+    huggingface::tgi::backends::TensorRtLlmBackend backend(engines, executor);
+}
diff --git a/backends/v2/Cargo.toml b/backends/v2/Cargo.toml
new file mode 100644
index 00000000..4d32474e
--- /dev/null
+++ b/backends/v2/Cargo.toml
@@ -0,0 +1,75 @@
+[package]
+name = "text-generation-router-v2"
+description = "Text Generation Webserver"
+version.workspace = true
+edition.workspace = true
+authors.workspace = true
+homepage.workspace = true
+
+[lib]
+path = "src/lib.rs"
+
+[[bin]]
+name = "text-generation-router-v2"
+path = "src/main.rs"
+
+[dependencies]
+async-trait = "0.1.74"
+async-stream = "0.3.5"
+axum = { version = "0.7", features = ["json"] }
+axum-tracing-opentelemetry = "0.16"
+text-generation-router = { path = "../../router" }
+clap = { version = "4.4.5", features = ["derive", "env"] }
+grpc-metadata = { path = "../grpc-metadata" }
+futures = "0.3.28"
+hf-hub = { workspace = true }
+jsonschema = { version = "0.17.1", features = ["draft202012"] }
+metrics = { workspace = true }
+metrics-exporter-prometheus = { workspace = true }
+nohash-hasher = "0.2.0"
+opentelemetry = { version = "0.20.0", features = ["rt-tokio"] }
+opentelemetry-otlp = "0.13.0"
+rand = "0.8.5"
+reqwest = { version = "0.11.20", features = [] }
+serde = "1.0.188"
+serde_json = "1.0.107"
+slotmap = "1.0.7"
+thiserror = "1.0.48"
+tokenizers = { workspace = true }
+tokio = { version = "1.32.0", features = [
+  "rt",
+  "rt-multi-thread",
+  "parking_lot",
+  "signal",
+  "sync",
+] }
+tokio-stream = "0.1.14"
+tower-http = { version = "0.5.1", features = ["cors"] }
+tracing = "0.1.37"
+tracing-opentelemetry = "0.21.0"
+tracing-subscriber = { version = "0.3.17", features = ["json", "env-filter"] }
+utoipa = { version = "4.2.0", features = ["axum_extras"] }
+utoipa-swagger-ui = { version = "6.0.0", features = ["axum"] }
+init-tracing-opentelemetry = { version = "0.14.1", features = [
+  "opentelemetry-otlp",
+] }
+minijinja = { workspace = true }
+minijinja-contrib = { workspace = true }
+futures-util = "0.3.30"
+regex = "1.10.3"
+once_cell = "1.19.0"
+image = "0.25.1"
+base64 = { workspace = true }
+prost = "^0.12"
+tonic = "^0.10"
+tower = "^0.4"
+
+[build-dependencies]
+tonic-build = "0.10.1"
+prost-build = "0.12.1"
+
+[features]
+default = ["ngrok"]
+ngrok = ["text-generation-router/ngrok"]
+google = ["text-generation-router/google"]
+kserve = ["text-generation-router/kserve"]
diff --git a/router/client/build.rs b/backends/v2/build.rs
similarity index 76%
rename from router/client/build.rs
rename to backends/v2/build.rs
index 497be545..f1d85dc7 100644
--- a/router/client/build.rs
+++ b/backends/v2/build.rs
@@ -1,16 +1,16 @@
 use std::fs;
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
-    println!("cargo:rerun-if-changed=../../proto/generate.proto");
-    fs::create_dir("src/pb").unwrap_or(());
+    println!("cargo:rerun-if-changed=../../proto/");
 
+    fs::create_dir_all("src/client/pb").unwrap_or(());
     let mut config = prost_build::Config::new();
     config.protoc_arg("--experimental_allow_proto3_optional");
 
     tonic_build::configure()
         .build_client(true)
         .build_server(false)
-        .out_dir("src/pb")
+        .out_dir("src/client/pb")
         .include_file("mod.rs")
         .compile_with_config(config, &["../../proto/generate.proto"], &["../../proto"])
         .unwrap_or_else(|e| panic!("protobuf compilation failed: {e}"));
diff --git a/backends/v2/src/backend.rs b/backends/v2/src/backend.rs
new file mode 100644
index 00000000..1ab582eb
--- /dev/null
+++ b/backends/v2/src/backend.rs
@@ -0,0 +1,517 @@
+use crate::client::{Batch, CachedBatch, ClientError, Generation, Health, ShardedClient};
+/// Batching and inference logic
+use crate::queue::{Entry, Queue};
+use async_trait::async_trait;
+use nohash_hasher::IntMap;
+use std::sync::Arc;
+use text_generation_router::infer::{Backend, GeneratedText, InferError, InferStreamResponse};
+use text_generation_router::validation::ValidGenerateRequest;
+use text_generation_router::{Attention, FinishReason, PrefillToken, Token};
+use tokio::sync::mpsc::error::SendError;
+use tokio::sync::{mpsc, Notify};
+use tokio::time::Instant;
+use tokio_stream::wrappers::UnboundedReceiverStream;
+use tracing::{info_span, instrument, Instrument, Span};
+
+pub struct BackendV2 {
+    /// Request queue
+    queue: Queue,
+    /// Notify batcher on queue appends
+    batching_task_notifier: Arc<Notify>,
+    /// Client clone, used for health checks to skip the queue
+    client: ShardedClient,
+}
+
+impl BackendV2 {
+    #[allow(clippy::too_many_arguments)]
+    pub(crate) fn new(
+        client: ShardedClient,
+        waiting_served_ratio: f32,
+        max_input_tokens: u32,
+        max_total_tokens: u32,
+        max_batch_prefill_tokens: u32,
+        max_batch_total_tokens: u32,
+        max_waiting_tokens: usize,
+        max_batch_size: Option<usize>,
+        requires_padding: bool,
+        window_size: Option<u32>,
+        speculate: u32,
+    ) -> Self {
+        // Infer shared state
+        let attention = if let Ok(attention) = std::env::var("ATTENTION") {
+            attention
+                .parse()
+                .unwrap_or_else(|_| panic!("Invalid attention was specified :`{attention}`"))
+        } else {
+            Attention::Paged
+        };
+        let block_size = if attention == Attention::FlashDecoding {
+            256
+        } else {
+            16
+        };
+
+        let queue = Queue::new(
+            requires_padding,
+            block_size,
+            window_size,
+            speculate,
+            max_input_tokens,
+            max_total_tokens,
+        );
+
+        let batching_task_notifier = Arc::new(Notify::new());
+
+        // Spawn batching background task that contains all the inference logic
+        tokio::spawn(batching_task(
+            client.clone(),
+            waiting_served_ratio,
+            max_batch_prefill_tokens,
+            max_batch_total_tokens,
+            max_waiting_tokens,
+            max_batch_size,
+            queue.clone(),
+            batching_task_notifier.clone(),
+        ));
+
+        Self {
+            queue,
+            batching_task_notifier,
+            client,
+        }
+    }
+}
+
+#[async_trait]
+impl Backend for BackendV2 {
+    #[instrument(skip_all)]
+    fn schedule(
+        &self,
+        request: ValidGenerateRequest,
+    ) -> Result<UnboundedReceiverStream<Result<InferStreamResponse, InferError>>, InferError> {
+        // MPSC channel to communicate with the background batching task
+        let (response_tx, response_rx) = mpsc::unbounded_channel();
+
+        // Append the request to the queue
+        self.queue.append(Entry {
+            request,
+            response_tx,
+            span: Span::current(),
+            temp_span: None,
+            queue_time: Instant::now(),
+            batch_time: None,
+        });
+
+        // Notify the background task that we have a new entry in the queue that needs
+        // to be batched
+        self.batching_task_notifier.notify_one();
+
+        // Return stream
+        Ok(UnboundedReceiverStream::new(response_rx))
+    }
+
+    async fn health(&self, current_health: bool) -> bool {
+        if current_health {
+            // Generation is healthy, we only check that the shards can allocate on device
+            self.client.device_health().await
+        } else {
+            self.client.model_health().await
+        }
+        .is_ok()
+    }
+}
+
+/// Batching logic
+/// Will be launched in a background Tokio task
+///
+/// Batches requests and sends them to the inference server
+#[allow(clippy::too_many_arguments)]
+pub(crate) async fn batching_task(
+    mut client: ShardedClient,
+    waiting_served_ratio: f32,
+    max_batch_prefill_tokens: u32,
+    max_batch_total_tokens: u32,
+    max_waiting_tokens: usize,
+    max_batch_size: Option<usize>,
+    queue: Queue,
+    notifier: Arc<Notify>,
+) {
+    // Infinite loop
+    loop {
+        // Wait for a notification from the Infer struct
+        notifier.notified().await;
+
+        // Get the next batch from the queue
+        // This batch might be smaller than the maximum batch size if there are not enough requests
+        // waiting in the queue
+        while let Some((mut entries, batch, span)) = queue
+            .next_batch(
+                None,
+                max_batch_size,
+                max_batch_prefill_tokens,
+                max_batch_total_tokens,
+            )
+            .await
+        {
+            let mut cached_batch = prefill(&mut client, batch, &mut entries)
+                .instrument(span)
+                .await;
+            let mut waiting_tokens = 1;
+
+            // We loop until we do not receive any cached batch from the inference server (== until
+            // all requests have met their stopping criteria)
+            while let Some(batch) = cached_batch {
+                // Get current batch info
+                let batch_size = batch.size;
+                let batch_max_tokens = batch.max_tokens;
+                let mut batches = vec![batch];
+                metrics::gauge!("tgi_batch_current_size").set(batch_size as f64);
+                metrics::gauge!("tgi_batch_current_max_tokens").set(batch_max_tokens as f64);
+
+                let min_size = if waiting_tokens >= max_waiting_tokens {
+                    // If we didn't onboard any new requests since >= max_waiting_tokens, we try
+                    // to add a new batch even though its size might be small
+                    None
+                } else {
+                    // Minimum batch size
+                    Some((batch_size as f32 * waiting_served_ratio).floor() as usize)
+                };
+
+                let token_budget = max_batch_total_tokens.saturating_sub(batch_max_tokens);
+                let max_size =
+                    max_batch_size.map(|max_size| max_size.saturating_sub(batch_size as usize));
+                // Try to get a new batch
+                if let Some((mut new_entries, new_batch, span)) = queue
+                    .next_batch(min_size, max_size, max_batch_prefill_tokens, token_budget)
+                    .await
+                {
+                    // Tracking metrics
+                    if min_size.is_some() {
+                        metrics::counter!("tgi_batch_concat", "reason" => "backpressure")
+                            .increment(1);
+                    } else {
+                        metrics::counter!("tgi_batch_concat", "reason" => "wait_exceeded")
+                            .increment(1);
+                    }
+
+                    entries.iter_mut().for_each(|(_, entry)| {
+                        // Create a new span to add the info that this entry is waiting
+                        // because a new batch is being computed
+                        let entry_waiting_span = info_span!(parent: &entry.span, "waiting");
+                        // Add relationships
+                        span.follows_from(&entry_waiting_span);
+                        entry_waiting_span.follows_from(&span);
+                        // Update entry
+                        entry.temp_span = Some(entry_waiting_span);
+                    });
+
+                    // Generate one token for this new batch to have the attention past in cache
+                    let new_cached_batch = prefill(&mut client, new_batch, &mut new_entries)
+                        .instrument(span)
+                        .await;
+                    // Reset waiting counter
+                    waiting_tokens = 1;
+                    // Extend current batch with the new batch
+                    if let Some(new_cached_batch) = new_cached_batch {
+                        entries.extend(new_entries);
+                        batches.push(new_cached_batch);
+                    }
+                }
+
+                // Create span for this batch to add context to inference calls
+                let next_batch_size = entries.len();
+                let next_batch_span =
+                    info_span!(parent: None, "batch", batch_size = next_batch_size);
+                entries.iter_mut().for_each(|(_, entry)| {
+                    // Create a new span to link the batch back to this entry
+                    let entry_batch_span = info_span!(parent: &entry.span, "infer");
+                    // Add relationships
+                    next_batch_span.follows_from(&entry_batch_span);
+                    entry_batch_span.follows_from(&next_batch_span);
+                    // Update entry
+                    entry.temp_span = Some(entry_batch_span);
+                });
+
+                cached_batch = decode(&mut client, batches, &mut entries)
+                    .instrument(next_batch_span)
+                    .await;
+                waiting_tokens += 1;
+            }
+            metrics::gauge!("tgi_batch_current_size").set(0.0);
+            metrics::gauge!("tgi_batch_current_max_tokens").set(0.0);
+        }
+    }
+}
+
+#[instrument(skip_all)]
+async fn prefill(
+    client: &mut ShardedClient,
+    batch: Batch,
+    entries: &mut IntMap<u64, Entry>,
+) -> Option<CachedBatch> {
+    let start_time = Instant::now();
+    let batch_id = batch.id;
+    metrics::counter!("tgi_batch_inference_count", "method" => "prefill").increment(1);
+
+    match client.prefill(batch).await {
+        Ok((generations, next_batch, timings)) => {
+            let start_filtering_time = Instant::now();
+            // Send generated tokens and filter stopped entries
+            filter_send_generations(generations, entries);
+
+            // Filter next batch and remove requests that were stopped
+            let next_batch = filter_batch(client, next_batch, entries).await;
+
+            metrics::histogram!("tgi_batch_forward_duration","method" => "prefill")
+                .record(timings.forward.as_secs_f64());
+            metrics::histogram!("tgi_batch_decode_duration", "method" => "prefill")
+                .record(timings.decode.as_secs_f64());
+            metrics::histogram!("tgi_batch_filter_duration", "method" => "prefill")
+                .record(start_filtering_time.elapsed().as_secs_f64());
+            metrics::histogram!("tgi_batch_inference_duration","method" => "prefill")
+                .record(start_time.elapsed().as_secs_f64());
+            metrics::counter!("tgi_batch_inference_success", "method" => "prefill").increment(1);
+            next_batch
+        }
+        // If we have an error, we discard the whole batch
+        Err(err) => {
+            let _ = client.clear_cache(Some(batch_id)).await;
+            send_errors(err, entries);
+            metrics::counter!("tgi_batch_inference_failure", "method" => "prefill").increment(1);
+            None
+        }
+    }
+}
+
+#[instrument(skip_all)]
+async fn decode(
+    client: &mut ShardedClient,
+    batches: Vec<CachedBatch>,
+    entries: &mut IntMap<u64, Entry>,
+) -> Option<CachedBatch> {
+    let start_time = Instant::now();
+    let batch_ids: Vec<u64> = batches.iter().map(|b| b.id).collect();
+    metrics::counter!("tgi_batch_inference_count", "method" => "decode").increment(1);
+
+    match client.decode(batches).await {
+        Ok((generations, next_batch, timings)) => {
+            let start_filtering_time = Instant::now();
+            // Send generated tokens and filter stopped entries
+            filter_send_generations(generations, entries);
+
+            // Filter next batch and remove requests that were stopped
+            let next_batch = filter_batch(client, next_batch, entries).await;
+
+            if let Some(concat_duration) = timings.concat {
+                metrics::histogram!("tgi_batch_concat_duration", "method" => "decode")
+                    .record(concat_duration.as_secs_f64());
+            }
+            metrics::histogram!("tgi_batch_forward_duration", "method" => "decode")
+                .record(timings.forward.as_secs_f64());
+            metrics::histogram!("tgi_batch_decode_duration", "method" => "decode")
+                .record(timings.decode.as_secs_f64());
+            metrics::histogram!("tgi_batch_filter_duration", "method" => "decode")
+                .record(start_filtering_time.elapsed().as_secs_f64());
+            metrics::histogram!("tgi_batch_inference_duration", "method" => "decode")
+                .record(start_time.elapsed().as_secs_f64());
+            metrics::counter!("tgi_batch_inference_success", "method" => "decode").increment(1);
+            next_batch
+        }
+        // If we have an error, we discard the whole batch
+        Err(err) => {
+            for id in batch_ids {
+                let _ = client.clear_cache(Some(id)).await;
+            }
+            send_errors(err, entries);
+            metrics::counter!("tgi_batch_inference_failure", "method" => "decode").increment(1);
+            None
+        }
+    }
+}
+
+/// Filter a `batch` and remove all requests not present in `entries`
+#[instrument(skip_all)]
+async fn filter_batch(
+    client: &mut ShardedClient,
+    next_batch: Option<CachedBatch>,
+    entries: &IntMap<u64, Entry>,
+) -> Option<CachedBatch> {
+    let mut batch = next_batch?;
+
+    // No need to filter
+    if batch.size as usize == entries.len() {
+        return Some(batch);
+    }
+
+    let id = batch.id;
+
+    // Retain only requests that are still in entries
+    batch.request_ids.retain(|id| entries.contains_key(id));
+
+    if batch.request_ids.is_empty() {
+        // All requests have been filtered out
+        // Next batch is now empty
+        // Clear it from the Python shards cache
+        // We unwrap here as we need to panic since we cannot recover if this method fails
+        client.clear_cache(Some(id)).await.unwrap();
+        None
+    } else {
+        // Filter Python shard cache
+        // We unwrap here as we need to panic since we cannot recover if this method fails
+        client.filter_batch(id, batch.request_ids).await.unwrap()
+    }
+}
+
+/// Send one or multiple `InferStreamResponse` to Infer for all `entries`
+/// and filter entries
+#[instrument(skip_all)]
+fn filter_send_generations(generations: Vec<Generation>, entries: &mut IntMap<u64, Entry>) {
+    generations.into_iter().for_each(|generation| {
+        let id = generation.request_id;
+        // Get entry
+        // We can `expect` here as the request id should always be in the entries
+        let entry = entries
+            .get(&id)
+            .expect("ID not found in entries. This is a bug.");
+
+        // Create and enter a span to link this function back to the entry
+        let _span = info_span!(parent: entry.temp_span.as_ref().expect("batch_span is None. This is a bug."), "send_generation", generation = ?generation).entered();
+        // Send generation responses back to the infer task
+        // If the receive an error from the Flume channel, it means that the client dropped the
+        // request and we need to stop generating hence why we unwrap_or(true)
+        let stopped = send_responses(generation, entry).inspect_err(|_err| {
+            tracing::error!("Entry response channel error.");
+            metrics::counter!("tgi_request_failure", "err" => "dropped").increment(1);
+        }).unwrap_or(true);
+        if stopped {
+            entries.remove(&id).expect("ID not found in entries. This is a bug.");
+        }
+    });
+}
+
+/// Send responses through the `entry` response channel
+fn send_responses(
+    generation: Generation,
+    entry: &Entry,
+) -> Result<bool, Box<SendError<Result<InferStreamResponse, InferError>>>> {
+    // Return directly if the channel is disconnected
+    if entry.response_tx.is_closed() {
+        metrics::counter!("tgi_request_failure", "err" => "dropped").increment(1);
+        return Ok(true);
+    }
+
+    let mut stopped = false;
+
+    if let Some(prefill_tokens) = generation.prefill_tokens {
+        // Create Token objects
+        // We do that here instead of in the Python code as Rust for loops are faster
+        let prefill_tokens = prefill_tokens
+            .ids
+            .into_iter()
+            .zip(prefill_tokens.logprobs)
+            .zip(prefill_tokens.texts)
+            .map(|((id, logprob), text)| PrefillToken { id, text, logprob })
+            .collect();
+
+        // Send message
+        entry
+            .response_tx
+            .send(Ok(InferStreamResponse::Prefill(prefill_tokens)))?;
+    }
+
+    // Create last Token
+    let tokens_ = generation.tokens.expect("Non empty tokens in generation");
+    let n = tokens_.ids.len();
+    metrics::histogram!("tgi_request_skipped_tokens").record((n - 1) as f64);
+    let mut iterator = tokens_
+        .ids
+        .into_iter()
+        .zip(tokens_.logprobs)
+        .zip(tokens_.texts)
+        .zip(tokens_.is_special)
+        .enumerate()
+        .peekable();
+    while let Some((i, (((id, logprob), text), special))) = iterator.next() {
+        let token = Token {
+            id,
+            text,
+            logprob,
+            special,
+        };
+        let top_tokens = if let Some(top_tokens_) = generation.top_tokens.get(i) {
+            top_tokens_
+                .ids
+                .iter()
+                .zip(top_tokens_.logprobs.iter())
+                .zip(top_tokens_.texts.iter())
+                .zip(top_tokens_.is_special.iter())
+                .map(|(((&id, &logprob), text), &special)| Token {
+                    id,
+                    text: text.to_string(),
+                    logprob,
+                    special,
+                })
+                .collect()
+        } else {
+            vec![]
+        };
+        match (&generation.generated_text, iterator.peek()) {
+            (Some(generated_text), None) => {
+                // Generation has ended
+                stopped = true;
+                // Send message
+                entry.response_tx.send(Ok(InferStreamResponse::End {
+                    token,
+                    top_tokens,
+                    generated_text: GeneratedText::from(generated_text.clone()),
+                    queued: entry.queue_time,
+                    start: entry.batch_time.unwrap(),
+                }))?;
+            }
+            _ => {
+                // Send message
+                entry
+                    .response_tx
+                    .send(Ok(InferStreamResponse::Intermediate { token, top_tokens }))?;
+            }
+        }
+    }
+
+    Ok(stopped)
+}
+
+/// Send errors to Infer for all `entries`
+#[instrument(skip_all)]
+fn send_errors(error: ClientError, entries: &mut IntMap<u64, Entry>) {
+    entries.drain().for_each(|(_, entry)| {
+        // Create and enter a span to link this function back to the entry
+        let _send_error_span = info_span!(parent: entry.temp_span.as_ref().expect("batch_span is None. This is a bug."), "send_error").entered();
+        let err = InferError::GenerationError(error.to_string());
+        metrics::counter!("tgi_request_failure", "err" => "generation").increment(1);
+        tracing::error!("{err}");
+
+        // unwrap_or is valid here as we don't care if the receiver is gone.
+        entry
+            .response_tx
+            .send(Err(err))
+            .unwrap_or(());
+    });
+}
+
+impl From<crate::client::GeneratedText> for GeneratedText {
+    fn from(value: crate::client::GeneratedText) -> Self {
+        let v2_finish_reason = crate::client::FinishReason::try_from(value.finish_reason).unwrap();
+        let finish_reason = match v2_finish_reason {
+            crate::client::FinishReason::Length => FinishReason::Length,
+            crate::client::FinishReason::EosToken => FinishReason::EndOfSequenceToken,
+            crate::client::FinishReason::StopSequence => FinishReason::StopSequence,
+        };
+
+        Self {
+            text: value.text,
+            generated_tokens: value.generated_tokens,
+            finish_reason,
+            seed: value.seed,
+        }
+    }
+}
diff --git a/backends/v2/src/client/grpc_client.rs b/backends/v2/src/client/grpc_client.rs
new file mode 100644
index 00000000..a56b8a54
--- /dev/null
+++ b/backends/v2/src/client/grpc_client.rs
@@ -0,0 +1,259 @@
+/// Single shard Client
+use crate::client::pb;
+use crate::client::{ClientError, Result, WARMUP_IMAGE_BASE64};
+use grpc_metadata::InjectTelemetryContext;
+use pb::generate::v2::text_generation_service_client::TextGenerationServiceClient;
+use pb::generate::v2::*;
+use std::cmp::min;
+use std::time::Duration;
+use tonic::transport::{Channel, Uri};
+use tracing::instrument;
+
+/// Text Generation Inference gRPC client
+#[derive(Debug, Clone)]
+pub struct Client {
+    stub: TextGenerationServiceClient<Channel>,
+}
+
+impl Client {
+    /// Returns a client connected to the given url
+    #[allow(dead_code)]
+    pub async fn connect(uri: Uri) -> Result<Self> {
+        let channel = Channel::builder(uri).connect().await?;
+
+        Ok(Self {
+            stub: TextGenerationServiceClient::new(channel),
+        })
+    }
+
+    /// Returns a client connected to the given unix socket
+    pub async fn connect_uds(path: String) -> Result<Self> {
+        let channel = Channel::from_shared("http://[::]:50051".to_string())
+            .unwrap()
+            .connect_with_connector(tower::service_fn(move |_: Uri| {
+                tokio::net::UnixStream::connect(path.clone())
+            }))
+            .await?;
+
+        Ok(Self {
+            stub: TextGenerationServiceClient::new(channel),
+        })
+    }
+
+    /// Returns a list of uris or unix sockets of all shards
+    #[instrument(skip(self))]
+    pub async fn service_discovery(&mut self) -> Result<Vec<String>> {
+        let request = tonic::Request::new(ServiceDiscoveryRequest {}).inject_context();
+        let response = self.stub.service_discovery(request).await.map_err(|_| {
+            ClientError::Connection("Server does not support v2 interface".to_string())
+        })?;
+        let urls = response
+            .into_inner()
+            .urls
+            .into_iter()
+            // Remove unix socket prefix
+            .map(|url| match url.strip_prefix("unix://") {
+                None => url,
+                Some(stripped_url) => stripped_url.to_string(),
+            })
+            .collect();
+        Ok(urls)
+    }
+
+    /// Get model info
+    #[instrument(skip(self))]
+    pub async fn info(&mut self) -> Result<InfoResponse> {
+        let request = tonic::Request::new(InfoRequest {}).inject_context();
+        let response = self.stub.info(request).await?.into_inner();
+        Ok(response)
+    }
+
+    /// Get model health
+    #[instrument(skip(self))]
+    pub async fn health(&mut self) -> Result<HealthResponse> {
+        let request = tonic::Request::new(HealthRequest {}).inject_context();
+        let response = self.stub.health(request).await?.into_inner();
+        Ok(response)
+    }
+
+    /// Clear the past generations cache
+    #[instrument(skip(self))]
+    pub async fn clear_cache(&mut self, batch_id: Option<u64>) -> Result<()> {
+        let request = tonic::Request::new(ClearCacheRequest { id: batch_id }).inject_context();
+        self.stub.clear_cache(request).await?;
+        Ok(())
+    }
+
+    /// Filter a cached batch
+    #[instrument(skip(self))]
+    pub async fn filter_batch(
+        &mut self,
+        batch_id: u64,
+        request_ids: Vec<u64>,
+    ) -> Result<Option<CachedBatch>> {
+        let request = tonic::Request::new(FilterBatchRequest {
+            batch_id,
+            request_ids,
+        })
+        .inject_context();
+        let filtered_batch = self.stub.filter_batch(request).await?.into_inner();
+        Ok(filtered_batch.batch)
+    }
+
+    /// Warmup on a max size batch
+    ///
+    /// Returns the maximum amount of tokens supported by the hardware
+    #[instrument(skip_all)]
+    pub async fn warmup(
+        &mut self,
+        max_input_length: u32,
+        max_prefill_tokens: u32,
+        max_total_tokens: u32,
+        max_batch_total_tokens: u32,
+        max_batch_size: Option<usize>,
+    ) -> Result<Option<u32>> {
+        let mut n_tokens = 0;
+        let mut requests = Vec::new();
+        // Create requests
+        while n_tokens < max_prefill_tokens {
+            let truncate = min(max_input_length, max_prefill_tokens - n_tokens);
+
+            let mut inputs = String::new();
+            inputs.push_str(&"_test ".to_string().repeat(max_input_length as usize));
+            if n_tokens == 0 {
+                // 1 request is enough to test vision heads.
+                // Sending images on other queries messes up easily with truncation.
+                inputs.push_str(&format!(
+                    "![](data:image/jpeg;base64,{WARMUP_IMAGE_BASE64})",
+                ));
+            }
+
+            requests.push(Request {
+                id: 0,
+                inputs,
+                // We truncate the input on the server side to be sure that it has the correct size
+                truncate,
+                // Set sampling parameters to also take these ops into account in the max memory
+                parameters: Some(NextTokenChooserParameters {
+                    temperature: 0.9,
+                    top_k: 10,
+                    top_p: 0.9,
+                    typical_p: 0.9,
+                    do_sample: false,
+                    seed: 0,
+                    repetition_penalty: 1.2,
+                    frequency_penalty: 0.1,
+                    watermark: true,
+                    grammar: String::new(),
+                    grammar_type: GrammarType::None as i32,
+                }),
+                stopping_parameters: Some(StoppingCriteriaParameters {
+                    max_new_tokens: max_total_tokens - truncate,
+                    stop_sequences: vec![],
+                    ignore_eos_token: true,
+                }),
+                prefill_logprobs: true,
+                top_n_tokens: 20,
+            });
+            n_tokens += max_input_length;
+
+            // Check max_batch_size
+            if Some(requests.len()) == max_batch_size {
+                break;
+            }
+        }
+
+        let batch = Batch {
+            id: 0,
+            size: requests.len() as u32,
+            requests,
+            max_tokens: 0,
+        };
+
+        let request = tonic::Request::new(WarmupRequest {
+            batch: Some(batch),
+            max_input_length,
+            max_prefill_tokens,
+            max_total_tokens,
+            max_batch_total_tokens,
+        })
+        .inject_context();
+        let response = self.stub.warmup(request).await?.into_inner();
+        Ok(response.max_supported_total_tokens)
+    }
+
+    /// Generate one token for each request in the given batch
+    ///
+    /// Returns Generation for each request in batch
+    /// and the next cached batch
+    #[instrument(skip_all, fields(id = &batch.id, size = &batch.size))]
+    pub async fn prefill(
+        &mut self,
+        batch: Batch,
+    ) -> Result<(Vec<Generation>, Option<CachedBatch>, PrefillTimings)> {
+        let request = tonic::Request::new(PrefillRequest { batch: Some(batch) }).inject_context();
+        let response = self.stub.prefill(request).await?.into_inner();
+        Ok((
+            response.generations,
+            response.batch,
+            PrefillTimings::new(response.forward_ns, response.decode_ns, response.total_ns),
+        ))
+    }
+
+    /// Generate one token for each request in the given cached batches
+    ///
+    /// Returns Generation for each request in batches
+    /// and the next cached batch
+    #[instrument(skip_all, fields(size = batches.iter().map(|batch|{batch.size}).sum::<u32>()))]
+    pub async fn decode(
+        &mut self,
+        batches: Vec<CachedBatch>,
+    ) -> Result<(Vec<Generation>, Option<CachedBatch>, DecodeTimings)> {
+        let request = tonic::Request::new(DecodeRequest { batches }).inject_context();
+        let response = self.stub.decode(request).await?.into_inner();
+        Ok((
+            response.generations,
+            response.batch,
+            DecodeTimings::new(
+                response.concat_ns,
+                response.forward_ns,
+                response.decode_ns,
+                response.total_ns,
+            ),
+        ))
+    }
+}
+
+pub struct PrefillTimings {
+    pub forward: Duration,
+    pub decode: Duration,
+    pub total: Duration,
+}
+
+impl PrefillTimings {
+    fn new(forward_ns: u64, decode_ns: u64, total_ns: u64) -> Self {
+        Self {
+            forward: Duration::from_nanos(forward_ns),
+            decode: Duration::from_nanos(decode_ns),
+            total: Duration::from_nanos(total_ns),
+        }
+    }
+}
+
+pub struct DecodeTimings {
+    pub concat: Option<Duration>,
+    pub forward: Duration,
+    pub decode: Duration,
+    pub total: Duration,
+}
+
+impl DecodeTimings {
+    fn new(concat_ns: Option<u64>, forward_ns: u64, decode_ns: u64, total_ns: u64) -> Self {
+        Self {
+            concat: concat_ns.map(Duration::from_nanos),
+            forward: Duration::from_nanos(forward_ns),
+            decode: Duration::from_nanos(decode_ns),
+            total: Duration::from_nanos(total_ns),
+        }
+    }
+}
diff --git a/backends/v2/src/client/mod.rs b/backends/v2/src/client/mod.rs
new file mode 100644
index 00000000..fa9d4406
--- /dev/null
+++ b/backends/v2/src/client/mod.rs
@@ -0,0 +1,68 @@
+//! Text Generation gRPC client library
+
+use async_trait::async_trait;
+use thiserror::Error;
+use tonic::transport;
+use tonic::Status;
+
+#[allow(clippy::derive_partial_eq_without_eq)]
+mod pb;
+
+mod grpc_client;
+mod sharded_client;
+
+pub use grpc_client::Client;
+pub use pb::generate::v2::{
+    Batch, CachedBatch, FinishReason, GeneratedText, Generation, GrammarType, HealthResponse,
+    InfoResponse, NextTokenChooserParameters, Request, StoppingCriteriaParameters,
+};
+pub use sharded_client::ShardedClient;
+
+#[async_trait]
+pub trait Health {
+    /// Check if a generate server is healthy by asking it to allocate a tensor on device
+    async fn device_health(&self) -> Result<()>;
+
+    /// Check if a generate server is healthy by doing a forward pass.
+    /// EXPENSIVE
+    async fn model_health(&self) -> Result<()>;
+}
+
+#[derive(Debug)]
+pub struct ShardInfo {
+    pub requires_padding: bool,
+    pub dtype: String,
+    pub device_type: String,
+    pub window_size: Option<u32>,
+    pub speculate: u32,
+}
+
+#[derive(Error, Debug, Clone)]
+pub enum ClientError {
+    #[error("Could not connect to Text Generation server: {0}")]
+    Connection(String),
+    #[error("Server error: {0}")]
+    Generation(String),
+    #[error("Sharded results are empty")]
+    EmptyResults,
+}
+
+impl From<Status> for ClientError {
+    fn from(err: Status) -> Self {
+        let err = Self::Generation(err.message().to_string());
+        tracing::error!("{err}");
+        err
+    }
+}
+
+impl From<transport::Error> for ClientError {
+    fn from(err: transport::Error) -> Self {
+        let err = Self::Connection(err.to_string());
+        tracing::error!("{err}");
+        err
+    }
+}
+
+static WARMUP_IMAGE_BASE64 :&str = "iVBORw0KGgoAAAANSUhEUgAAABQAAAAUCAIAAAAC64paAAABg2lDQ1BJQ0MgcHJvZmlsZQAAKJF9kT1Iw0AcxV/TSotUROxQxCFDdbKLijjWKhShQqgVWnUwufQLmrQkKS6OgmvBwY/FqoOLs64OroIg+AHi7OCk6CIl/i8ptIjx4Lgf7+497t4BQqvKNDOQADTdMjKppJjLr4rBVwQQwhAERGVm1uckKQ3P8XUPH1/v4jzL+9yfY0AtmAzwicQJVjcs4g3imU2rznmfOMLKskp8Tjxh0AWJH7muuPzGueSwwDMjRjYzTxwhFks9rPQwKxsa8TRxTNV0yhdyLquctzhr1Qbr3JO/MFzQV5a5TnMUKSxiCRJEKGiggiosxGnVSTGRof2kh3/E8UvkUshVASPHAmrQIDt+8D/43a1ZnJp0k8JJoO/Ftj/GgOAu0G7a9vexbbdPAP8zcKV3/bUWMPtJerOrxY6AwW3g4rqrKXvA5Q4QfarLhuxIfppCsQi8n9E35YHhW6B/ze2ts4/TByBLXaVvgINDYLxE2ese7w719vbvmU5/PycecohsjayNAAAACXBIWXMAAC4jAAAuIwF4pT92AAAAB3RJTUUH6AQIEQMnlTSSjwAAABl0RVh0Q29tbWVudABDcmVhdGVkIHdpdGggR0lNUFeBDhcAAAASSURBVDjLY2AYBaNgFIyCoQsABMQAAeRw1DoAAAAASUVORK5CYII=";
+
+pub type Result<T> = std::result::Result<T, ClientError>;
diff --git a/backends/v2/src/client/sharded_client.rs b/backends/v2/src/client/sharded_client.rs
new file mode 100644
index 00000000..238bd773
--- /dev/null
+++ b/backends/v2/src/client/sharded_client.rs
@@ -0,0 +1,254 @@
+/// Multi shard Client
+use crate::client::{ClientError, Result};
+use crate::client::{Health, ShardInfo};
+
+use crate::client::grpc_client::{DecodeTimings, PrefillTimings};
+use crate::client::InfoResponse;
+use crate::client::{
+    Batch, CachedBatch, Client, Generation, GrammarType, HealthResponse,
+    NextTokenChooserParameters, Request, StoppingCriteriaParameters,
+};
+use async_trait::async_trait;
+use futures::future::join_all;
+use tonic::transport::Uri;
+use tracing::instrument;
+
+#[derive(Debug, Clone)]
+/// Text Generation Inference gRPC multi client
+pub struct ShardedClient {
+    clients: Vec<Client>,
+}
+
+impl ShardedClient {
+    fn new(clients: Vec<Client>) -> Self {
+        Self { clients }
+    }
+
+    /// Create a new ShardedClient from a master client. The master client will communicate with
+    /// the other shards and returns all uris/unix sockets with the `service_discovery` gRPC method.
+    async fn from_master_client(mut master_client: Client) -> Result<Self> {
+        // Get all uris/unix sockets from the master client
+        let uris = master_client.service_discovery().await?;
+        let futures = uris.into_iter().map(Client::connect_uds);
+        let clients: Result<Vec<Client>> = join_all(futures).await.into_iter().collect();
+        Ok(Self::new(clients?))
+    }
+
+    /// Returns a client connected to the given uri
+    #[allow(dead_code)]
+    pub async fn connect(uri: Uri) -> Result<Self> {
+        let master_client = Client::connect(uri).await?;
+        Self::from_master_client(master_client).await
+    }
+
+    /// Returns a client connected to the given unix socket
+    pub async fn connect_uds(path: String) -> Result<Self> {
+        let master_client = Client::connect_uds(path).await?;
+        Self::from_master_client(master_client).await
+    }
+
+    /// Get the model info
+    #[instrument(skip(self))]
+    pub async fn info(&mut self) -> Result<ShardInfo> {
+        let futures: Vec<_> = self
+            .clients
+            .iter_mut()
+            .map(|client| client.info())
+            .collect();
+        join_all(futures).await.pop().unwrap().map(ShardInfo::from)
+    }
+
+    /// GRPC health check
+    #[instrument(skip(self))]
+    pub async fn health(&mut self) -> Result<HealthResponse> {
+        let futures: Vec<_> = self
+            .clients
+            .iter_mut()
+            .map(|client| client.health())
+            .collect();
+        join_all(futures).await.pop().unwrap()
+    }
+
+    /// Clear the past generations cache
+    #[instrument(skip(self))]
+    pub async fn clear_cache(&mut self, batch_id: Option<u64>) -> Result<()> {
+        let futures: Vec<_> = self
+            .clients
+            .iter_mut()
+            .map(|client| client.clear_cache(batch_id))
+            .collect();
+        join_all(futures).await.into_iter().collect()
+    }
+
+    /// Filter a cached batch
+    #[instrument(skip(self))]
+    pub async fn filter_batch(
+        &mut self,
+        batch_id: u64,
+        request_ids: Vec<u64>,
+    ) -> Result<Option<CachedBatch>> {
+        let futures: Vec<_> = self
+            .clients
+            .iter_mut()
+            .map(|client| Box::pin(client.filter_batch(batch_id, request_ids.clone())))
+            .collect();
+        // all shards return the same message
+        join_all(futures).await.pop().unwrap()
+    }
+
+    /// Warmup on a max size batch
+    ///
+    /// Returns the maximum amount of tokens supported by the hardware
+    #[instrument(skip(self))]
+    pub async fn warmup(
+        &mut self,
+        max_input_length: u32,
+        max_prefill_tokens: u32,
+        max_total_tokens: u32,
+        max_batch_total_tokens: u32,
+        max_batch_size: Option<usize>,
+    ) -> Result<Option<u32>> {
+        let futures: Vec<_> = self
+            .clients
+            .iter_mut()
+            .map(|client| {
+                Box::pin(client.warmup(
+                    max_input_length,
+                    max_prefill_tokens,
+                    max_total_tokens,
+                    max_batch_total_tokens,
+                    max_batch_size,
+                ))
+            })
+            .collect();
+        // Take the minimum value
+        let results = join_all(futures)
+            .await
+            .into_iter()
+            .collect::<Result<Vec<Option<u32>>>>()?;
+        Ok(results.into_iter().flatten().min())
+    }
+
+    /// Generate one token for each request in the given batch
+    ///
+    /// Returns Generation for each request in batch
+    /// and the next cached batch
+    #[instrument(skip_all, fields(id = & batch.id, size = & batch.size))]
+    pub async fn prefill(
+        &mut self,
+        batch: Batch,
+    ) -> Result<(Vec<Generation>, Option<CachedBatch>, PrefillTimings)> {
+        let futures: Vec<_> = self
+            .clients
+            .iter_mut()
+            .map(|client| Box::pin(client.prefill(batch.clone())))
+            .collect();
+        #[allow(clippy::type_complexity)]
+        let results: Result<Vec<(Vec<Generation>, Option<CachedBatch>, PrefillTimings)>> =
+            join_all(futures).await.into_iter().collect();
+        let mut results = results?;
+
+        let (mut generations, next_batch, mut timings) =
+            results.pop().ok_or(ClientError::EmptyResults)?;
+
+        // Merge generations from different model shards
+        for (mut shard_generations, _, shard_timings) in results.into_iter() {
+            generations.append(&mut shard_generations);
+            // Return the timings of the slowest shard
+            if shard_timings.total > timings.total {
+                timings = shard_timings;
+            }
+        }
+        Ok((generations, next_batch, timings))
+    }
+
+    /// Generate one token for each request in the given cached batches
+    ///
+    /// Returns Generation for each request in batches
+    /// and the next cached batch
+    #[instrument(skip_all, fields(size = batches.iter().map(| batch | {batch.size}).sum::< u32 > ()))]
+    pub async fn decode(
+        &mut self,
+        batches: Vec<CachedBatch>,
+    ) -> Result<(Vec<Generation>, Option<CachedBatch>, DecodeTimings)> {
+        let futures: Vec<_> = self
+            .clients
+            .iter_mut()
+            .map(|client| Box::pin(client.decode(batches.clone())))
+            .collect();
+        #[allow(clippy::type_complexity)]
+        let results: Result<Vec<(Vec<Generation>, Option<CachedBatch>, DecodeTimings)>> =
+            join_all(futures).await.into_iter().collect();
+        let mut results = results?;
+
+        let (mut generations, next_batch, mut timings) =
+            results.pop().ok_or(ClientError::EmptyResults)?;
+
+        // Merge generations from different model shards
+        for (mut shard_generations, _, shard_timings) in results.into_iter() {
+            generations.append(&mut shard_generations);
+            // Return the timings of the slowest shard
+            if shard_timings.total > timings.total {
+                timings = shard_timings;
+            }
+        }
+        Ok((generations, next_batch, timings))
+    }
+}
+
+impl From<InfoResponse> for ShardInfo {
+    fn from(value: InfoResponse) -> Self {
+        Self {
+            requires_padding: value.requires_padding,
+            dtype: value.dtype,
+            device_type: value.device_type,
+            window_size: value.window_size,
+            speculate: value.speculate,
+        }
+    }
+}
+
+#[async_trait]
+impl Health for ShardedClient {
+    async fn device_health(&self) -> Result<()> {
+        self.clone().health().await?;
+        Ok(())
+    }
+
+    async fn model_health(&self) -> Result<()> {
+        // Dummy batch of 1 token and 1 generated token
+        let liveness_request = Request {
+            id: u64::MAX,
+            inputs: "liveness".to_string(),
+            truncate: 10,
+            prefill_logprobs: false,
+            parameters: Some(NextTokenChooserParameters {
+                temperature: 1.0,
+                top_k: 0,
+                top_p: 1.0,
+                typical_p: 1.0,
+                do_sample: false,
+                seed: 0,
+                repetition_penalty: 1.0,
+                frequency_penalty: 0.0,
+                watermark: false,
+                grammar: String::new(),
+                grammar_type: GrammarType::None as i32,
+            }),
+            stopping_parameters: Some(StoppingCriteriaParameters {
+                max_new_tokens: 1,
+                stop_sequences: vec![],
+                ignore_eos_token: false,
+            }),
+            top_n_tokens: 0,
+        };
+        let batch = Batch {
+            id: u64::MAX,
+            requests: vec![liveness_request],
+            size: 1,
+            max_tokens: 2,
+        };
+        self.clone().prefill(batch).await?;
+        Ok(())
+    }
+}
diff --git a/backends/v2/src/lib.rs b/backends/v2/src/lib.rs
new file mode 100644
index 00000000..90f03230
--- /dev/null
+++ b/backends/v2/src/lib.rs
@@ -0,0 +1,144 @@
+mod backend;
+mod client;
+mod queue;
+
+use crate::client::{ClientError, ShardedClient};
+pub(crate) use backend::BackendV2;
+use serde::Serialize;
+use thiserror::Error;
+use utoipa::ToSchema;
+
+#[derive(Clone, Debug, Serialize, ToSchema)]
+pub struct BackendInfo {
+    /// Mandatory
+    #[schema(example = "cuda")]
+    pub model_device_type: String,
+    #[schema(example = "torch.float16")]
+    pub model_dtype: String,
+
+    /// Backend parameters
+    #[schema(example = "1")]
+    pub speculate: usize,
+    #[schema(example = "1.2")]
+    pub waiting_served_ratio: f32,
+    #[schema(example = "32000")]
+    pub max_batch_total_tokens: u32,
+    #[schema(example = "20")]
+    pub max_waiting_tokens: usize,
+    #[schema(nullable = true, example = "null")]
+    pub max_batch_size: Option<usize>,
+}
+
+#[allow(clippy::too_many_arguments)]
+pub async fn connect_backend(
+    max_input_tokens: usize,
+    max_total_tokens: usize,
+    master_shard_uds_path: String,
+    waiting_served_ratio: f32,
+    max_batch_prefill_tokens: u32,
+    max_batch_total_tokens: Option<u32>,
+    max_waiting_tokens: usize,
+    max_batch_size: Option<usize>,
+) -> Result<(BackendV2, BackendInfo), V2Error> {
+    // Helper function
+    let check_max_batch_total_tokens = |max_supported_batch_total_tokens: Option<u32>| {
+        match max_supported_batch_total_tokens {
+            // Older models do not support automatic max-batch-total-tokens
+            None => {
+                let max_batch_total_tokens = max_batch_total_tokens
+                    .unwrap_or(16000.max((max_total_tokens as u32).max(max_batch_prefill_tokens)));
+                tracing::warn!("Model does not support automatic max batch total tokens");
+                Ok(max_batch_total_tokens)
+            }
+            // Flash attention models return their max supported total tokens
+            Some(max_supported_batch_total_tokens) => {
+                // Warn if user added his own max-batch-total-tokens as we will ignore it
+                if max_batch_total_tokens.is_some() {
+                    tracing::warn!(
+                        "`--max-batch-total-tokens` is deprecated for Flash \
+                        Attention models."
+                    );
+                    tracing::warn!(
+                        "Inferred max batch total tokens: {max_supported_batch_total_tokens}"
+                    );
+                }
+                if max_total_tokens as u32 > max_supported_batch_total_tokens {
+                    return Err(V2Error::NotEnoughMemory(max_total_tokens));
+                }
+
+                Ok(max_supported_batch_total_tokens)
+            }
+        }
+    };
+
+    let mut sharded_client = ShardedClient::connect_uds(master_shard_uds_path)
+        .await
+        .map_err(V2Error::Connection)?;
+
+    // server is running on v2
+    // Clear the cache; useful if the webserver rebooted
+    sharded_client
+        .clear_cache(None)
+        .await
+        .map_err(V2Error::Cache)?;
+    // Get info from the shard
+    let shard_info = sharded_client.info().await.map_err(V2Error::Info)?;
+
+    // Warmup model
+    tracing::info!("Warming up model");
+    let max_batch_total_tokens = check_max_batch_total_tokens(
+        sharded_client
+            .warmup(
+                max_input_tokens as u32,
+                max_batch_prefill_tokens,
+                max_total_tokens as u32,
+                max_batch_total_tokens.unwrap_or(16000.max((max_total_tokens as u32).max(max_batch_prefill_tokens))),
+                max_batch_size,
+            )
+            .await
+            .map_err(V2Error::Warmup)?,
+    )?;
+    tracing::info!("Setting max batch total tokens to {max_batch_total_tokens}");
+
+    let backend_info = BackendInfo {
+        waiting_served_ratio,
+        max_batch_total_tokens,
+        max_waiting_tokens,
+        max_batch_size,
+        model_device_type: shard_info.device_type.clone(),
+        model_dtype: shard_info.dtype.clone(),
+        speculate: shard_info.speculate as usize,
+    };
+
+    let backend = BackendV2::new(
+        sharded_client,
+        waiting_served_ratio,
+        max_input_tokens as u32,
+        max_total_tokens as u32,
+        max_batch_prefill_tokens,
+        max_batch_total_tokens,
+        max_waiting_tokens,
+        max_batch_size,
+        shard_info.requires_padding,
+        shard_info.window_size,
+        shard_info.speculate,
+    );
+
+    tracing::info!("Using backend V3");
+
+    Ok((backend, backend_info))
+}
+
+#[derive(Debug, Error)]
+pub enum V2Error {
+    #[error("Unable to clear the Python model shards cache: {0}")]
+    Cache(ClientError),
+    #[error("Unable to connect to the Python model shards: {0}")]
+    Connection(ClientError),
+    #[error("Unable to get the Python model shards info: {0}")]
+    Info(ClientError),
+    #[error("Unable to warmup the Python model shards: {0}")]
+    Warmup(ClientError),
+    #[error("Not enough memory to handle `max_total_tokens={0}`")]
+    NotEnoughMemory(usize),
+}
diff --git a/backends/v2/src/main.rs b/backends/v2/src/main.rs
new file mode 100644
index 00000000..f53d898e
--- /dev/null
+++ b/backends/v2/src/main.rs
@@ -0,0 +1,212 @@
+use clap::{Parser, Subcommand};
+use text_generation_router::{server, usage_stats};
+use text_generation_router_v2::{connect_backend, V2Error};
+use thiserror::Error;
+
+/// App Configuration
+#[derive(Parser, Debug)]
+#[clap(author, version, about, long_about = None)]
+struct Args {
+    #[command(subcommand)]
+    command: Option<Commands>,
+
+    #[clap(default_value = "128", long, env)]
+    max_concurrent_requests: usize,
+    #[clap(default_value = "2", long, env)]
+    max_best_of: usize,
+    #[clap(default_value = "4", long, env)]
+    max_stop_sequences: usize,
+    #[clap(default_value = "5", long, env)]
+    max_top_n_tokens: u32,
+    #[clap(default_value = "1024", long, env)]
+    max_input_tokens: usize,
+    #[clap(default_value = "2048", long, env)]
+    max_total_tokens: usize,
+    #[clap(default_value = "1.2", long, env)]
+    waiting_served_ratio: f32,
+    #[clap(default_value = "4096", long, env)]
+    max_batch_prefill_tokens: u32,
+    #[clap(long, env)]
+    max_batch_total_tokens: Option<u32>,
+    #[clap(default_value = "20", long, env)]
+    max_waiting_tokens: usize,
+    #[clap(long, env)]
+    max_batch_size: Option<usize>,
+    #[clap(default_value = "0.0.0.0", long, env)]
+    hostname: String,
+    #[clap(default_value = "3000", long, short, env)]
+    port: u16,
+    #[clap(default_value = "/tmp/text-generation-server-0", long, env)]
+    master_shard_uds_path: String,
+    #[clap(default_value = "bigscience/bloom", long, env)]
+    tokenizer_name: String,
+    #[clap(long, env)]
+    tokenizer_config_path: Option<String>,
+    #[clap(long, env)]
+    revision: Option<String>,
+    #[clap(default_value = "2", long, env)]
+    validation_workers: usize,
+    #[clap(long, env)]
+    api_key: Option<String>,
+    #[clap(long, env)]
+    json_output: bool,
+    #[clap(long, env)]
+    otlp_endpoint: Option<String>,
+    #[clap(default_value = "text-generation-inference.router", long, env)]
+    otlp_service_name: String,
+    #[clap(long, env)]
+    cors_allow_origin: Option<Vec<String>>,
+    #[clap(long, env)]
+    ngrok: bool,
+    #[clap(long, env)]
+    ngrok_authtoken: Option<String>,
+    #[clap(long, env)]
+    ngrok_edge: Option<String>,
+    #[clap(long, env, default_value_t = false)]
+    messages_api_enabled: bool,
+    #[clap(long, env, default_value_t = false)]
+    disable_grammar_support: bool,
+    #[clap(default_value = "4", long, env)]
+    max_client_batch_size: usize,
+    #[clap(default_value = "on", long, env)]
+    usage_stats: usage_stats::UsageStatsLevel,
+}
+
+#[derive(Debug, Subcommand)]
+enum Commands {
+    PrintSchema,
+}
+
+#[tokio::main]
+async fn main() -> Result<(), RouterError> {
+    // Get args
+    let args = Args::parse();
+    // Pattern match configuration
+    let Args {
+        command,
+        max_concurrent_requests,
+        max_best_of,
+        max_stop_sequences,
+        max_top_n_tokens,
+        max_input_tokens,
+        max_total_tokens,
+        waiting_served_ratio,
+        max_batch_prefill_tokens,
+        max_batch_total_tokens,
+        max_waiting_tokens,
+        max_batch_size,
+        hostname,
+        port,
+        master_shard_uds_path,
+        tokenizer_name,
+        tokenizer_config_path,
+        revision,
+        validation_workers,
+        api_key,
+        json_output,
+        otlp_endpoint,
+        otlp_service_name,
+        cors_allow_origin,
+        ngrok,
+        ngrok_authtoken,
+        ngrok_edge,
+        messages_api_enabled,
+        disable_grammar_support,
+        max_client_batch_size,
+        usage_stats,
+    } = args;
+
+    if let Some(Commands::PrintSchema) = command {
+        use utoipa::OpenApi;
+        let api_doc = text_generation_router::server::ApiDoc::openapi();
+        let api_doc = serde_json::to_string_pretty(&api_doc).unwrap();
+        println!("{}", api_doc);
+        std::process::exit(0);
+    };
+    text_generation_router::logging::init_logging(otlp_endpoint, otlp_service_name, json_output);
+
+    // Validate args
+    if max_input_tokens >= max_total_tokens {
+        return Err(RouterError::ArgumentValidation(
+            "`max_input_tokens` must be < `max_total_tokens`".to_string(),
+        ));
+    }
+    if max_input_tokens as u32 > max_batch_prefill_tokens {
+        return Err(RouterError::ArgumentValidation(format!("`max_batch_prefill_tokens` must be >= `max_input_tokens`. Given: {max_batch_prefill_tokens} and {max_input_tokens}")));
+    }
+
+    if validation_workers == 0 {
+        return Err(RouterError::ArgumentValidation(
+            "`validation_workers` must be > 0".to_string(),
+        ));
+    }
+
+    if let Some(ref max_batch_total_tokens) = max_batch_total_tokens {
+        if max_batch_prefill_tokens > *max_batch_total_tokens {
+            return Err(RouterError::ArgumentValidation(format!("`max_batch_prefill_tokens` must be <= `max_batch_total_tokens`. Given: {max_batch_prefill_tokens} and {max_batch_total_tokens}")));
+        }
+        if max_total_tokens as u32 > *max_batch_total_tokens {
+            return Err(RouterError::ArgumentValidation(format!("`max_total_tokens` must be <= `max_batch_total_tokens`. Given: {max_total_tokens} and {max_batch_total_tokens}")));
+        }
+    }
+
+    if let Some(max_batch_size) = max_batch_size {
+        if max_batch_size == 0 {
+            return Err(RouterError::ArgumentValidation(
+                "`max_batch_size` must be > 0".to_string(),
+            ));
+        }
+    }
+
+    let (backend, _backend_info) = connect_backend(
+        max_input_tokens,
+        max_total_tokens,
+        master_shard_uds_path,
+        waiting_served_ratio,
+        max_batch_prefill_tokens,
+        max_batch_total_tokens,
+        max_waiting_tokens,
+        max_batch_size,
+    )
+    .await?;
+
+    // Run server
+    server::run(
+        backend,
+        max_concurrent_requests,
+        max_best_of,
+        max_stop_sequences,
+        max_top_n_tokens,
+        max_input_tokens,
+        max_total_tokens,
+        validation_workers,
+        api_key,
+        tokenizer_name,
+        tokenizer_config_path,
+        revision,
+        hostname,
+        port,
+        cors_allow_origin,
+        ngrok,
+        ngrok_authtoken,
+        ngrok_edge,
+        messages_api_enabled,
+        disable_grammar_support,
+        max_client_batch_size,
+        usage_stats,
+    )
+    .await?;
+    Ok(())
+}
+
+#[derive(Debug, Error)]
+enum RouterError {
+    #[error("Argument validation error: {0}")]
+    ArgumentValidation(String),
+    #[error("Backend failed: {0}")]
+    Backend(#[from] V2Error),
+    #[error("WebServer error: {0}")]
+    WebServer(#[from] server::WebServerError),
+    #[error("Tokio runtime failed to start: {0}")]
+    Tokio(#[from] std::io::Error),
+}
diff --git a/router/src/queue.rs b/backends/v2/src/queue.rs
similarity index 76%
rename from router/src/queue.rs
rename to backends/v2/src/queue.rs
index 11690bf7..5f793d09 100644
--- a/router/src/queue.rs
+++ b/backends/v2/src/queue.rs
@@ -1,15 +1,14 @@
-/// Copyright (C) 2024 Habana Labs, Ltd. an Intel Company.
-
-use crate::infer::InferError;
-use crate::infer::InferStreamResponse;
-use crate::validation::ValidGenerateRequest;
+use crate::client::{
+    Batch, GrammarType, NextTokenChooserParameters, Request, StoppingCriteriaParameters,
+};
 use nohash_hasher::{BuildNoHashHasher, IntMap};
 use std::cmp::min;
-use std::cmp::{Eq, Ord, PartialEq, PartialOrd};
-use std::collections::BinaryHeap;
-use std::env;
-use std::time::Duration;
-use text_generation_client::{Batch, Request};
+use std::collections::VecDeque;
+use text_generation_router::infer::InferError;
+use text_generation_router::infer::InferStreamResponse;
+use text_generation_router::validation::{
+    ChunksToString, ValidGenerateRequest, ValidGrammar, ValidParameters, ValidStoppingParameters,
+};
 use tokio::sync::{mpsc, oneshot};
 use tokio::time::Instant;
 use tracing::{info_span, instrument, Span};
@@ -41,11 +40,11 @@ pub(crate) struct Queue {
 impl Queue {
     pub(crate) fn new(
         requires_padding: bool,
-        max_input_length: u32,
-        max_total_tokens: u32,
         block_size: u32,
         window_size: Option<u32>,
         speculate: u32,
+        max_input_tokens: u32,
+        max_total_tokens: u32,
     ) -> Self {
         // Create channel
         let (queue_sender, queue_receiver) = mpsc::unbounded_channel();
@@ -53,18 +52,17 @@ impl Queue {
         // Launch background queue task
         tokio::spawn(queue_task(
             requires_padding,
-            max_input_length,
-            max_total_tokens,
             block_size,
             window_size,
             speculate,
+            max_input_tokens,
+            max_total_tokens,
             queue_receiver,
         ));
 
         Self { queue_sender }
     }
 
-    /// Append an entry to the queue
     #[instrument(skip_all)]
     pub(crate) fn append(&self, entry: Entry) {
         // Send append command to the background task managing the state
@@ -106,27 +104,27 @@ impl Queue {
 // Background task responsible of the queue state
 async fn queue_task(
     requires_padding: bool,
-    max_input_length: u32,
-    max_total_tokens: u32,
     block_size: u32,
     window_size: Option<u32>,
     speculate: u32,
+    max_input_tokens: u32,
+    max_total_tokens: u32,
     mut receiver: mpsc::UnboundedReceiver<QueueCommand>,
 ) {
     let mut state = State::new(
         requires_padding,
-        max_input_length,
-        max_total_tokens,
         block_size,
         window_size,
-        speculate
+        speculate,
+        max_input_tokens,
+        max_total_tokens,
     );
 
     while let Some(cmd) = receiver.recv().await {
         match cmd {
             QueueCommand::Append(entry, span) => {
                 span.in_scope(|| state.append(*entry));
-                metrics::increment_gauge!("tgi_queue_size", 1.0);
+                metrics::gauge!("tgi_queue_size").increment(1.0);
             }
             QueueCommand::NextBatch {
                 min_size,
@@ -139,110 +137,17 @@ async fn queue_task(
                 let next_batch =
                     state.next_batch(min_size, max_size, prefill_token_budget, token_budget);
                 response_sender.send(next_batch).unwrap();
-                metrics::gauge!("tgi_queue_size", state.entries.len() as f64);
+                metrics::gauge!("tgi_queue_size").set(state.entries.len() as f64);
             }),
         }
     }
 }
 
-#[derive(Debug)]
-struct IdentifiableEntry(u64, Entry);
-
-impl Eq for IdentifiableEntry {}
-
-impl PartialEq for IdentifiableEntry {
-    fn eq(&self, other: &Self) -> bool {
-        self.0 == other.0
-    }
-}
-
-impl Ord for IdentifiableEntry {
-    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
-        let ordering = match self
-            .1
-            .request
-            .input_length
-            .cmp(&other.1.request.input_length)
-        {
-            std::cmp::Ordering::Equal => self.0.cmp(&other.0),
-            any => any,
-        };
-
-        // inverse to get min heap
-        return ordering.reverse();
-    }
-}
-
-impl PartialOrd for IdentifiableEntry {
-    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
-        Some(self.cmp(other))
-    }
-}
-
-#[derive(Debug)]
-struct QueueImpl {
-    regular_entries: BinaryHeap<IdentifiableEntry>,
-    overdue_entries: BinaryHeap<IdentifiableEntry>,
-    overdue_threshold: Duration,
-}
-
-impl QueueImpl {
-    fn new(capacity: usize, overdue_threshold: Duration) -> Self {
-        Self {
-            regular_entries: BinaryHeap::with_capacity(capacity),
-            overdue_entries: BinaryHeap::with_capacity(capacity),
-            overdue_threshold,
-        }
-    }
-
-    fn update(&mut self) {
-        if self.regular_entries.is_empty() {
-            return;
-        }
-
-        let mut left = BinaryHeap::with_capacity(self.regular_entries.capacity());
-
-        for entry in self.regular_entries.drain() {
-            if entry.1.queue_time.elapsed() > self.overdue_threshold {
-                self.overdue_entries.push(entry);
-            } else {
-                left.push(entry);
-            }
-        }
-
-        self.regular_entries = left;
-    }
-
-    fn push(&mut self, entry: IdentifiableEntry) {
-        if entry.1.queue_time.elapsed() > self.overdue_threshold {
-            self.overdue_entries.push(entry);
-        } else {
-            self.regular_entries.push(entry);
-        }
-    }
-
-    fn pop(&mut self) -> Option<IdentifiableEntry> {
-        if !self.overdue_entries.is_empty() {
-            self.overdue_entries.pop()
-        } else {
-            self.regular_entries.pop()
-        }
-    }
-
-    fn is_empty(&self) -> bool {
-        self.regular_entries.is_empty() && self.overdue_entries.is_empty()
-    }
-
-    fn len(&self) -> usize {
-        self.regular_entries.len() + self.overdue_entries.len()
-    }
-}
-
 /// Queue State
 #[derive(Debug)]
 struct State {
-    /// Queue entries
-    entries: QueueImpl,
+    /// Queue entries organized in a Vec
+    entries: VecDeque<(u64, Entry)>,
 
     /// Id of the next entry
     next_id: u64,
@@ -253,12 +158,6 @@ struct State {
     /// Whether the model is using padding
     requires_padding: bool,
 
-    /// Maximum input length, required for padding scenario
-    max_input_length: u32,
-
-    /// Maximum input and output length, required for padding scenario
-    max_total_tokens: u32,
-
     /// Paged Attention block size
     block_size: u32,
 
@@ -267,33 +166,33 @@ struct State {
 
     /// Speculation amount
     speculate: u32,
+
+    /// max input tokens
+    max_input_tokens: u32,
+
+    /// max total tokens,
+    max_total_tokens: u32,
 }
 
 impl State {
     fn new(
         requires_padding: bool,
-        max_input_length: u32,
-        max_total_tokens: u32,
         block_size: u32,
         window_size: Option<u32>,
         speculate: u32,
+        max_input_tokens: u32,
+        max_total_tokens: u32,
     ) -> Self {
-        let default_threshold: u64 = 120;
-        let threshold: u64 = match env::var("QUEUE_THRESHOLD_MS") {
-            Ok(val) => val.parse().unwrap_or(default_threshold),
-            Err(_) => default_threshold,
-        };
-
         Self {
-            entries: QueueImpl::new(128, Duration::from_millis(threshold)),
+            entries: VecDeque::with_capacity(128),
             next_id: 0,
             next_batch_id: 0,
             requires_padding,
-            max_input_length,
-            max_total_tokens,
             block_size,
             window_size,
             speculate,
+            max_input_tokens,
+            max_total_tokens,
         }
     }
 
@@ -304,7 +203,7 @@ impl State {
         entry.temp_span = Some(queue_span);
 
         // Push entry in the queue
-        self.entries.push(IdentifiableEntry(self.next_id, entry));
+        self.entries.push_back((self.next_id, entry));
         self.next_id += 1;
     }
 
@@ -329,11 +228,20 @@ impl State {
             }
         }
 
-        self.entries.update();
+        if let Some(max_size) = max_size {
+            if max_size == 0 {
+                tracing::debug!("No capacity");
+                return None;
+            }
+        }
+
+        // Pad prefill_token_budget to be a multiple of block size
+        let prefill_token_budget =
+            ((prefill_token_budget + self.block_size - 1) / self.block_size) * self.block_size;
 
         // Create span for this batch to add context to inference calls
         let next_batch_span = info_span!(parent: None, "batch", batch_size = tracing::field::Empty);
-        next_batch_span.follows_from(&Span::current());
+        next_batch_span.follows_from(Span::current());
 
         let mut batch_requests = Vec::with_capacity(self.entries.len());
         let mut batch_entries =
@@ -343,11 +251,11 @@ impl State {
         let mut decode_tokens: u32 = 0;
 
         // Pop entries starting from the front of the queue
-        while let Some(IdentifiableEntry(id, mut entry)) = self.entries.pop() {
+        while let Some((id, mut entry)) = self.entries.pop_front() {
             // Filter entries where the response receiver was dropped (== entries where the request
             // was dropped by the client)
             if entry.response_tx.is_closed() {
-                metrics::increment_counter!("tgi_request_failure", "err" => "dropped");
+                metrics::counter!("tgi_request_failure", "err" => "dropped").increment(1);
                 tracing::debug!("Dropping entry");
                 continue;
             }
@@ -355,7 +263,7 @@ impl State {
             if self.requires_padding {
                 // We pad to max input length in the Python shards
                 // We need to take these padding tokens into the equation
-                prefill_tokens = (batch_requests.len() + 1) as u32 * self.max_input_length;
+                prefill_tokens = (batch_requests.len() + 1) as u32 * self.max_input_tokens
             } else {
                 // pad to block size
                 prefill_tokens += ((entry.request.input_length + self.block_size - 1)
@@ -364,9 +272,7 @@ impl State {
             }
 
             if self.requires_padding {
-                // We pad to max total tokens in the Python shards
-                // We need to take these padding tokens into the equation
-                decode_tokens = (batch_requests.len() + 1) as u32 * (self.max_total_tokens - self.max_input_length);
+                decode_tokens = (batch_requests.len() + 1) as u32 * (self.max_total_tokens - self.max_input_tokens);
             } else {
                 let max_new_tokens = match self.window_size {
                     None => entry.request.stopping_parameters.max_new_tokens,
@@ -387,7 +293,7 @@ impl State {
                 // Entry is over budget
                 // Add it back to the front
                 tracing::debug!("Over budget: prefill_tokens={prefill_tokens} > {prefill_token_budget} || {prefill_tokens} + {decode_tokens} + {} > {token_budget}", self.speculate);
-                self.entries.push(IdentifiableEntry(id, entry));
+                self.entries.push_front((id, entry));
                 break;
             }
 
@@ -403,10 +309,14 @@ impl State {
             batch_requests.push(Request {
                 id,
                 prefill_logprobs: entry.request.decoder_input_details,
-                inputs: entry.request.inputs.clone(),
+                inputs: entry.request.inputs.chunks_to_string(),
                 truncate: entry.request.truncate,
-                parameters: Some(entry.request.parameters.clone()),
-                stopping_parameters: Some(entry.request.stopping_parameters.clone()),
+                parameters: Some(NextTokenChooserParameters::from(
+                    entry.request.parameters.clone(),
+                )),
+                stopping_parameters: Some(StoppingCriteriaParameters::from(
+                    entry.request.stopping_parameters.clone(),
+                )),
                 top_n_tokens: entry.request.top_n_tokens,
             });
             // Set batch_time
@@ -422,7 +332,7 @@ impl State {
 
         // Empty batch
         if batch_requests.is_empty() {
-            tracing::debug!("Filterered out all entries");
+            tracing::debug!("Filtered out all entries");
             return None;
         }
 
@@ -434,7 +344,7 @@ impl State {
                 for r in batch_requests.into_iter().rev() {
                     let id = r.id;
                     let entry = batch_entries.remove(&id).unwrap();
-                    self.entries.push(IdentifiableEntry(id, entry));
+                    self.entries.push_front((id, entry));
                 }
 
                 return None;
@@ -454,7 +364,7 @@ impl State {
         // Increment batch id
         self.next_batch_id += 1;
 
-        metrics::histogram!("tgi_batch_next_size", batch.size as f64);
+        metrics::histogram!("tgi_batch_next_size").record(batch.size as f64);
 
         Some((batch_entries, batch, next_batch_span))
     }
@@ -475,26 +385,49 @@ enum QueueCommand {
     },
 }
 
+impl From<ValidParameters> for NextTokenChooserParameters {
+    fn from(value: ValidParameters) -> Self {
+        let (grammar, grammar_type) = match value.grammar {
+            None => (String::new(), GrammarType::None),
+
+            Some(grammar) => match grammar {
+                ValidGrammar::Json(grammar_string) => (grammar_string, GrammarType::Json),
+                ValidGrammar::Regex(grammar_string) => (grammar_string, GrammarType::Regex),
+            },
+        };
+
+        Self {
+            temperature: value.temperature,
+            top_k: value.top_k,
+            top_p: value.top_p,
+            typical_p: value.typical_p,
+            do_sample: value.do_sample,
+            seed: value.seed,
+            repetition_penalty: value.repetition_penalty,
+            frequency_penalty: value.frequency_penalty,
+            watermark: value.watermark,
+            grammar,
+            grammar_type: grammar_type.into(),
+        }
+    }
+}
+
+impl From<ValidStoppingParameters> for StoppingCriteriaParameters {
+    fn from(value: ValidStoppingParameters) -> Self {
+        Self {
+            max_new_tokens: value.max_new_tokens,
+            stop_sequences: value.stop_sequences,
+            ignore_eos_token: value.ignore_eos_token,
+        }
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
-    use text_generation_client::{
-        GrammarType as ProtoGrammarType, NextTokenChooserParameters, StoppingCriteriaParameters,
-    };
+    use std::sync::Arc;
     use tracing::info_span;
 
-    fn default_queue() -> Queue {
-        Queue::new(
-            true, 1, 2, 1, None, 0
-        )
-    }
-
-    fn default_state() -> State {
-        State::new(
-            true, 1, 2, 1, None, 0
-        )
-    }
-
     fn default_entry() -> (
         Entry,
         mpsc::UnboundedReceiver<Result<InferStreamResponse, InferError>>,
@@ -503,11 +436,13 @@ mod tests {
 
         let entry = Entry {
             request: ValidGenerateRequest {
-                inputs: String::new(),
+                inputs: vec![],
+                input_ids: Some(Arc::new(vec![])),
                 input_length: 0,
+                add_special_tokens: true,
                 truncate: 0,
                 decoder_input_details: false,
-                parameters: NextTokenChooserParameters {
+                parameters: ValidParameters {
                     temperature: 0.0,
                     top_k: 0,
                     top_p: 0.0,
@@ -517,15 +452,15 @@ mod tests {
                     repetition_penalty: 0.0,
                     frequency_penalty: 0.0,
                     watermark: false,
-                    grammar: String::new(),
-                    grammar_type: ProtoGrammarType::None as i32,
+                    grammar: None,
                 },
-                stopping_parameters: StoppingCriteriaParameters {
+                stopping_parameters: ValidStoppingParameters {
                     ignore_eos_token: false,
                     max_new_tokens: 1,
                     stop_sequences: vec![],
                 },
                 top_n_tokens: 0,
+                adapter_id: None,
             },
             response_tx,
             span: info_span!("entry"),
@@ -538,7 +473,7 @@ mod tests {
 
     #[test]
     fn test_append() {
-        let mut state = default_state();
+        let mut state = State::new(false, 1, None, 0);
         let (entry, _guard) = default_entry();
 
         assert_eq!(state.next_id, 0);
@@ -548,13 +483,13 @@ mod tests {
 
         assert_eq!(state.next_id, 1);
         assert_eq!(state.entries.len(), 1);
-        let id = state.entries.pop().unwrap().0;
+        let (id, _) = state.entries.remove(0).unwrap();
         assert_eq!(id, 0);
     }
 
     #[test]
     fn test_next_batch_empty() {
-        let mut state = default_state();
+        let mut state = State::new(false, 1, None, 0);
 
         assert!(state.next_batch(None, None, 1, 1).is_none());
         assert!(state.next_batch(Some(1), None, 1, 1).is_none());
@@ -562,13 +497,13 @@ mod tests {
 
     #[test]
     fn test_next_batch_min_size() {
-        let mut state = default_state();
+        let mut state = State::new(false, 1, None, 0);
         let (entry1, _guard1) = default_entry();
         let (entry2, _guard2) = default_entry();
         state.append(entry1);
         state.append(entry2);
 
-        let (entries, batch, _) = state.next_batch(None, None, 2, 4).unwrap();
+        let (entries, batch, _) = state.next_batch(None, None, 2, 2).unwrap();
         assert_eq!(entries.len(), 2);
         assert!(entries.contains_key(&0));
         assert!(entries.contains_key(&1));
@@ -588,13 +523,13 @@ mod tests {
 
         assert_eq!(state.next_id, 3);
         assert_eq!(state.entries.len(), 1);
-        let IdentifiableEntry(id, _) = state.entries.pop().unwrap();
+        let (id, _) = state.entries.remove(0).unwrap();
         assert_eq!(id, 2);
     }
 
     #[test]
     fn test_next_batch_max_size() {
-        let mut state = default_state();
+        let mut state = State::new(false, 1, None, 0);
         let (entry1, _guard1) = default_entry();
         let (entry2, _guard2) = default_entry();
         state.append(entry1);
@@ -614,13 +549,13 @@ mod tests {
 
     #[test]
     fn test_next_batch_token_budget() {
-        let mut state = default_state();
+        let mut state = State::new(false, 1, None, 0);
         let (entry1, _guard1) = default_entry();
         let (entry2, _guard2) = default_entry();
         state.append(entry1);
         state.append(entry2);
 
-        let (entries, batch, _) = state.next_batch(None, None, 1, 2).unwrap();
+        let (entries, batch, _) = state.next_batch(None, None, 1, 1).unwrap();
         assert_eq!(entries.len(), 1);
         assert!(entries.contains_key(&0));
         assert_eq!(batch.id, 0);
@@ -633,7 +568,7 @@ mod tests {
         let (entry3, _guard3) = default_entry();
         state.append(entry3);
 
-        let (entries, batch, _) = state.next_batch(None, None, 3, 6).unwrap();
+        let (entries, batch, _) = state.next_batch(None, None, 3, 3).unwrap();
         assert_eq!(entries.len(), 2);
         assert!(entries.contains_key(&1));
         assert!(entries.contains_key(&2));
@@ -647,14 +582,14 @@ mod tests {
 
     #[tokio::test]
     async fn test_queue_append() {
-        let queue = default_queue();
+        let queue = Queue::new(false, 1, None, 0);
         let (entry, _guard) = default_entry();
         queue.append(entry);
     }
 
     #[tokio::test]
     async fn test_queue_next_batch_empty() {
-        let queue = default_queue();
+        let queue = Queue::new(false, 1, None, 0);
 
         assert!(queue.next_batch(None, None, 1, 1).await.is_none());
         assert!(queue.next_batch(Some(1), None, 1, 1).await.is_none());
@@ -662,13 +597,13 @@ mod tests {
 
     #[tokio::test]
     async fn test_queue_next_batch_min_size() {
-        let queue = default_queue();
+        let queue = Queue::new(false, 1, None, 0);
         let (entry1, _guard1) = default_entry();
         let (entry2, _guard2) = default_entry();
         queue.append(entry1);
         queue.append(entry2);
 
-        let (entries, batch, _) = queue.next_batch(None, None, 2, 4).await.unwrap();
+        let (entries, batch, _) = queue.next_batch(None, None, 2, 2).await.unwrap();
         assert_eq!(entries.len(), 2);
         assert!(entries.contains_key(&0));
         assert!(entries.contains_key(&1));
@@ -685,7 +620,7 @@ mod tests {
         // Not enough token budget
         assert!(queue.next_batch(Some(1), None, 0, 0).await.is_none());
         // Ok
-        let (entries2, batch2, _) = queue.next_batch(Some(1), None, 2, 4).await.unwrap();
+        let (entries2, batch2, _) = queue.next_batch(Some(1), None, 2, 2).await.unwrap();
         assert_eq!(entries2.len(), 1);
         assert!(entries2.contains_key(&2));
         assert!(entries2.get(&2).unwrap().batch_time.is_some());
@@ -695,7 +630,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_queue_next_batch_max_size() {
-        let queue = default_queue();
+        let queue = Queue::new(false, 1, None, 0);
         let (entry1, _guard1) = default_entry();
         let (entry2, _guard2) = default_entry();
         queue.append(entry1);
@@ -711,13 +646,13 @@ mod tests {
 
     #[tokio::test]
     async fn test_queue_next_batch_token_budget() {
-        let queue = default_queue();
+        let queue = Queue::new(false, 1, None, 0);
         let (entry1, _guard1) = default_entry();
         let (entry2, _guard2) = default_entry();
         queue.append(entry1);
         queue.append(entry2);
 
-        let (entries, batch, _) = queue.next_batch(None, None, 1, 2).await.unwrap();
+        let (entries, batch, _) = queue.next_batch(None, None, 1, 1).await.unwrap();
         assert_eq!(entries.len(), 1);
         assert!(entries.contains_key(&0));
         assert_eq!(batch.id, 0);
@@ -726,7 +661,7 @@ mod tests {
         let (entry3, _guard3) = default_entry();
         queue.append(entry3);
 
-        let (entries, batch, _) = queue.next_batch(None, None, 3, 6).await.unwrap();
+        let (entries, batch, _) = queue.next_batch(None, None, 3, 3).await.unwrap();
         assert_eq!(entries.len(), 2);
         assert!(entries.contains_key(&1));
         assert!(entries.contains_key(&2));
@@ -736,7 +671,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_queue_next_batch_token_speculate() {
-        let queue = Queue::new(true, 1, 2, 1, None, 2);
+        let queue = Queue::new(false, 1, None, 2);
         let (entry1, _guard1) = default_entry();
         let (entry2, _guard2) = default_entry();
         queue.append(entry1);
@@ -755,7 +690,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_queue_next_batch_dropped_receiver() {
-        let queue = default_queue();
+        let queue = Queue::new(false, 1, None, 0);
         let (entry, _) = default_entry();
         queue.append(entry);
 
diff --git a/backends/v3/Cargo.toml b/backends/v3/Cargo.toml
new file mode 100644
index 00000000..69dad072
--- /dev/null
+++ b/backends/v3/Cargo.toml
@@ -0,0 +1,83 @@
+[package]
+name = "text-generation-router-v3"
+description = "Text Generation Webserver"
+version.workspace = true
+edition.workspace = true
+authors.workspace = true
+homepage.workspace = true
+
+[lib]
+path = "src/lib.rs"
+
+[[bin]]
+name = "text-generation-router"
+path = "src/main.rs"
+
+[dependencies]
+async-trait = "0.1.74"
+async-stream = "0.3.5"
+axum = { version = "0.7", features = ["json"] }
+axum-tracing-opentelemetry = "0.16"
+text-generation-router = { path = "../../router" }
+clap = { version = "4.4.5", features = ["derive", "env"] }
+grpc-metadata = { path = "../grpc-metadata" }
+futures = "0.3.28"
+hf-hub = { workspace = true }
+jsonschema = { version = "0.17.1", features = ["draft202012"] }
+metrics = { workspace = true }
+metrics-exporter-prometheus = { workspace = true }
+nohash-hasher = "0.2.0"
+opentelemetry = { version = "0.20.0", features = ["rt-tokio"] }
+opentelemetry-otlp = "0.13.0"
+rand = "0.8.5"
+reqwest = { version = "0.11.20", features = [] }
+serde = "1.0.188"
+serde_json = "1.0.107"
+slotmap = "1.0.7"
+thiserror = "1.0.48"
+tokenizers = { workspace = true }
+tokio = { version = "1.32.0", features = [
+  "rt",
+  "rt-multi-thread",
+  "parking_lot",
+  "signal",
+  "sync",
+] }
+tokio-stream = "0.1.14"
+tower-http = { version = "0.5.1", features = ["cors"] }
+tracing = "0.1.37"
+tracing-opentelemetry = "0.21.0"
+tracing-subscriber = { version = "0.3.17", features = ["json", "env-filter"] }
+utoipa = { version = "4.2.0", features = ["axum_extras"] }
+utoipa-swagger-ui = { version = "6.0.0", features = ["axum"] }
+init-tracing-opentelemetry = { version = "0.14.1", features = [
+  "opentelemetry-otlp",
+] }
+minijinja = { workspace = true }
+minijinja-contrib = { workspace = true }
+futures-util = "0.3.30"
+regex = "1.10.3"
+once_cell = "1.19.0"
+image = "0.25.1"
+base64 = { workspace = true }
+prost = "^0.12"
+tonic = "^0.10"
+tower = "^0.4"
+
+[build-dependencies]
+tonic-build = "0.10.1"
+prost-build = "0.12.1"
+
+[dev-dependencies]
+criterion = "0.3"
+itertools = "0.13"
+
+[features]
+default = ["ngrok"]
+ngrok = ["text-generation-router/ngrok"]
+google = ["text-generation-router/google"]
+kserve = ["text-generation-router/kserve"]
+
+[[bench]]
+name = "prefix_cache"
+harness = false
diff --git a/backends/v3/benches/prefix_cache.rs b/backends/v3/benches/prefix_cache.rs
new file mode 100644
index 00000000..d9df45b2
--- /dev/null
+++ b/backends/v3/benches/prefix_cache.rs
@@ -0,0 +1,47 @@
+use std::sync::Arc;
+
+use criterion::{black_box, criterion_group, criterion_main, Criterion};
+use rand::Rng;
+
+use text_generation_router_v3::block_allocator::Allocator;
+use text_generation_router_v3::radix::RadixAllocator;
+
+fn prefix_cache_benchmark(c: &mut Criterion) {
+    // let prefixes: Vec<Vec<u32>> = (0..8192)
+    //     .chunks(256)
+    //     .into_iter()
+    //     .map(|c| c.collect())
+    //     .collect();
+
+    let mut cache = RadixAllocator::new(1, 262144, None);
+
+    c.bench_function("Radix allocator", |b| {
+        b.iter_batched(
+            || {
+                //prefixes
+                //    .choose_multiple(&mut rand::thread_rng(), 5)
+                //    .fold(Vec::new(), |mut v, s| {
+                //        v.extend(s);
+                //        v
+                //    })
+
+                (0..7936)
+                    .map(|_| rand::thread_rng().gen_range(0..1024))
+                    .collect::<Vec<u32>>()
+            },
+            |prefill| {
+                let alloc = cache.allocate(
+                    prefill.len() as u32 + 13,
+                    Some(Arc::new(black_box(prefill))),
+                );
+                if let Some(alloc) = alloc {
+                    cache.free(alloc.blocks.clone(), alloc.allocation_id);
+                }
+            },
+            criterion::BatchSize::SmallInput,
+        );
+    });
+}
+
+criterion_group!(benches, prefix_cache_benchmark);
+criterion_main!(benches);
diff --git a/backends/v3/build.rs b/backends/v3/build.rs
new file mode 100644
index 00000000..6d702d14
--- /dev/null
+++ b/backends/v3/build.rs
@@ -0,0 +1,19 @@
+use std::fs;
+
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    println!("cargo:rerun-if-changed=../../proto/");
+
+    fs::create_dir_all("src/client/pb").unwrap_or(());
+    let mut config = prost_build::Config::new();
+    config.protoc_arg("--experimental_allow_proto3_optional");
+
+    tonic_build::configure()
+        .build_client(true)
+        .build_server(false)
+        .out_dir("src/client/pb")
+        .include_file("mod.rs")
+        .compile_with_config(config, &["../../proto/v3/generate.proto"], &["../../proto"])
+        .unwrap_or_else(|e| panic!("protobuf compilation failed: {e}"));
+
+    Ok(())
+}
diff --git a/backends/v3/src/backend.rs b/backends/v3/src/backend.rs
new file mode 100644
index 00000000..b1268152
--- /dev/null
+++ b/backends/v3/src/backend.rs
@@ -0,0 +1,518 @@
+use crate::client::{Batch, CachedBatch, ClientError, Generation, Health, ShardedClient};
+/// Batching and inference logic
+use crate::queue::{Entry, Queue};
+use async_trait::async_trait;
+use nohash_hasher::IntMap;
+use std::sync::Arc;
+use text_generation_router::infer::{Backend, GeneratedText, InferError, InferStreamResponse};
+use text_generation_router::validation::ValidGenerateRequest;
+use text_generation_router::{Attention, FinishReason, PrefillToken, Token};
+use tokio::sync::mpsc::error::SendError;
+use tokio::sync::{mpsc, Notify};
+use tokio::time::Instant;
+use tokio_stream::wrappers::UnboundedReceiverStream;
+use tracing::{info_span, instrument, Instrument, Span};
+
+pub struct BackendV3 {
+    /// Request queue
+    queue: Queue,
+    /// Notify batcher on queue appends
+    batching_task_notifier: Arc<Notify>,
+    /// Client clone, used for health checks to skip the queue
+    client: ShardedClient,
+}
+
+impl BackendV3 {
+    #[allow(clippy::too_many_arguments)]
+    pub(crate) fn new(
+        client: ShardedClient,
+        waiting_served_ratio: f32,
+        max_input_tokens: u32,
+        max_total_tokens: u32,
+        max_batch_prefill_tokens: u32,
+        max_batch_total_tokens: u32,
+        max_waiting_tokens: usize,
+        max_batch_size: Option<usize>,
+        requires_padding: bool,
+        window_size: Option<u32>,
+        speculate: u32,
+    ) -> Self {
+        let prefix_caching =
+            std::env::var("USE_PREFIX_CACHING").expect("Expect prefix caching env var");
+        let prefix_caching = matches!(prefix_caching.as_str(), "true" | "1");
+        let attention: String = std::env::var("ATTENTION").expect("attention env var");
+
+        let attention: Attention = attention
+            .parse()
+            .unwrap_or_else(|_| panic!("Invalid attention was specified :`{attention}`"));
+        let block_size = attention.block_size();
+
+        let queue = Queue::new(
+            requires_padding,
+            block_size,
+            prefix_caching,
+            window_size,
+            speculate,
+            max_input_tokens,
+            max_total_tokens,
+            max_batch_total_tokens,
+        );
+        let batching_task_notifier = Arc::new(Notify::new());
+
+        // Spawn batching background task that contains all the inference logic
+        tokio::spawn(batching_task(
+            client.clone(),
+            waiting_served_ratio,
+            max_batch_prefill_tokens,
+            max_batch_total_tokens,
+            max_waiting_tokens,
+            max_batch_size,
+            queue.clone(),
+            batching_task_notifier.clone(),
+        ));
+
+        Self {
+            queue,
+            batching_task_notifier,
+            client,
+        }
+    }
+}
+
+#[async_trait]
+impl Backend for BackendV3 {
+    #[instrument(skip_all)]
+    fn schedule(
+        &self,
+        request: ValidGenerateRequest,
+    ) -> Result<UnboundedReceiverStream<Result<InferStreamResponse, InferError>>, InferError> {
+        // MPSC channel to communicate with the background batching task
+        let (response_tx, response_rx) = mpsc::unbounded_channel();
+
+        // Append the request to the queue
+        self.queue.append(Entry {
+            request,
+            response_tx,
+            span: Span::current(),
+            temp_span: None,
+            queue_time: Instant::now(),
+            batch_time: None,
+            block_allocation: None,
+        });
+
+        // Notify the background task that we have a new entry in the queue that needs
+        // to be batched
+        self.batching_task_notifier.notify_one();
+
+        // Return stream
+        Ok(UnboundedReceiverStream::new(response_rx))
+    }
+
+    async fn health(&self, current_health: bool) -> bool {
+        if current_health {
+            // Generation is healthy, we only check that the shards can allocate on device
+            self.client.device_health().await
+        } else {
+            self.client.model_health().await
+        }
+        .is_ok()
+    }
+}
+
+/// Batching logic
+/// Will be launched in a background Tokio task
+///
+/// Batches requests and sends them to the inference server
+#[allow(clippy::too_many_arguments)]
+pub(crate) async fn batching_task(
+    mut client: ShardedClient,
+    waiting_served_ratio: f32,
+    max_batch_prefill_tokens: u32,
+    max_batch_total_tokens: u32,
+    max_waiting_tokens: usize,
+    max_batch_size: Option<usize>,
+    queue: Queue,
+    notifier: Arc<Notify>,
+) {
+    // Infinite loop
+    loop {
+        // Wait for a notification from the Infer struct
+        notifier.notified().await;
+
+        // Get the next batch from the queue
+        // This batch might be smaller than the maximum batch size if there are not enough requests
+        // waiting in the queue
+        while let Some((mut entries, batch, span)) = queue
+            .next_batch(
+                None,
+                max_batch_size,
+                max_batch_prefill_tokens,
+                max_batch_total_tokens,
+            )
+            .await
+        {
+            let mut cached_batch = prefill(&mut client, batch, &mut entries)
+                .instrument(span)
+                .await;
+            let mut waiting_tokens = 1;
+
+            // We loop until we do not receive any cached batch from the inference server (== until
+            // all requests have met their stopping criteria)
+            while let Some(batch) = cached_batch {
+                // Get current batch info
+                let batch_size = batch.size;
+                let batch_max_tokens = batch.max_tokens;
+                let mut batches = vec![batch];
+                metrics::gauge!("tgi_batch_current_size").set(batch_size as f64);
+                metrics::gauge!("tgi_batch_current_max_tokens").set(batch_max_tokens as f64);
+
+                let min_size = if waiting_tokens >= max_waiting_tokens {
+                    // If we didn't onboard any new requests since >= max_waiting_tokens, we try
+                    // to add a new batch even though its size might be small
+                    None
+                } else {
+                    // Minimum batch size
+                    // TODO: temporarily disable to avoid incorrect deallocation +
+                    //       reallocation when using prefix caching.
+                    Some((batch_size as f32 * waiting_served_ratio).floor() as usize)
+                };
+
+                let token_budget = max_batch_total_tokens.saturating_sub(batch_max_tokens);
+                let max_size =
+                    max_batch_size.map(|max_size| max_size.saturating_sub(batch_size as usize));
+
+                // Try to get a new batch
+                if let Some((mut new_entries, new_batch, span)) = queue
+                    .next_batch(min_size, max_size, max_batch_prefill_tokens, token_budget)
+                    .await
+                {
+                    // Tracking metrics
+                    if min_size.is_some() {
+                        metrics::counter!("tgi_batch_concat", "reason" => "backpressure")
+                            .increment(1);
+                    } else {
+                        metrics::counter!("tgi_batch_concat", "reason" => "wait_exceeded")
+                            .increment(1);
+                    }
+
+                    entries.iter_mut().for_each(|(_, entry)| {
+                        // Create a new span to add the info that this entry is waiting
+                        // because a new batch is being computed
+                        let entry_waiting_span = info_span!(parent: &entry.span, "waiting");
+                        // Add relationships
+                        span.follows_from(&entry_waiting_span);
+                        entry_waiting_span.follows_from(&span);
+                        // Update entry
+                        entry.temp_span = Some(entry_waiting_span);
+                    });
+
+                    // Generate one token for this new batch to have the attention past in cache
+                    let new_cached_batch = prefill(&mut client, new_batch, &mut new_entries)
+                        .instrument(span)
+                        .await;
+                    // Reset waiting counter
+                    waiting_tokens = 1;
+                    // Extend current batch with the new batch
+                    if let Some(new_cached_batch) = new_cached_batch {
+                        entries.extend(new_entries);
+                        batches.push(new_cached_batch);
+                    }
+                }
+
+                // Create span for this batch to add context to inference calls
+                let next_batch_size = entries.len();
+                let next_batch_span =
+                    info_span!(parent: None, "batch", batch_size = next_batch_size);
+                entries.iter_mut().for_each(|(_, entry)| {
+                    // Create a new span to link the batch back to this entry
+                    let entry_batch_span = info_span!(parent: &entry.span, "infer");
+                    // Add relationships
+                    next_batch_span.follows_from(&entry_batch_span);
+                    entry_batch_span.follows_from(&next_batch_span);
+                    // Update entry
+                    entry.temp_span = Some(entry_batch_span);
+                });
+
+                cached_batch = decode(&mut client, batches, &mut entries)
+                    .instrument(next_batch_span)
+                    .await;
+                waiting_tokens += 1;
+            }
+            metrics::gauge!("tgi_batch_current_size").set(0.0);
+            metrics::gauge!("tgi_batch_current_max_tokens").set(0.0);
+        }
+    }
+}
+
+#[instrument(skip_all)]
+async fn prefill(
+    client: &mut ShardedClient,
+    batch: Batch,
+    entries: &mut IntMap<u64, Entry>,
+) -> Option<CachedBatch> {
+    let start_time = Instant::now();
+    let batch_id = batch.id;
+    metrics::counter!("tgi_batch_inference_count", "method" => "prefill").increment(1);
+
+    match client.prefill(batch).await {
+        Ok((generations, next_batch, timings)) => {
+            let start_filtering_time = Instant::now();
+            // Send generated tokens and filter stopped entries
+            filter_send_generations(generations, entries);
+
+            // Filter next batch and remove requests that were stopped
+            let next_batch = filter_batch(client, next_batch, entries).await;
+
+            metrics::histogram!("tgi_batch_forward_duration", "method" => "prefill")
+                .record(timings.forward.as_secs_f64());
+            metrics::histogram!("tgi_batch_decode_duration", "method" => "prefill")
+                .record(timings.decode.as_secs_f64());
+            metrics::histogram!("tgi_batch_filter_duration", "method" => "prefill")
+                .record(start_filtering_time.elapsed().as_secs_f64());
+            metrics::histogram!("tgi_batch_inference_duration", "method" => "prefill")
+                .record(start_time.elapsed().as_secs_f64());
+            metrics::counter!("tgi_batch_inference_success", "method" => "prefill").increment(1);
+            next_batch
+        }
+        // If we have an error, we discard the whole batch
+        Err(err) => {
+            let _ = client.clear_cache(Some(batch_id)).await;
+            send_errors(err, entries);
+            metrics::counter!("tgi_batch_inference_failure", "method" => "prefill").increment(1);
+            None
+        }
+    }
+}
+
+#[instrument(skip_all)]
+async fn decode(
+    client: &mut ShardedClient,
+    batches: Vec<CachedBatch>,
+    entries: &mut IntMap<u64, Entry>,
+) -> Option<CachedBatch> {
+    let start_time = Instant::now();
+    let batch_ids: Vec<u64> = batches.iter().map(|b| b.id).collect();
+    metrics::counter!("tgi_batch_inference_count", "method" => "decode").increment(1);
+
+    match client.decode(batches).await {
+        Ok((generations, next_batch, timings)) => {
+            let start_filtering_time = Instant::now();
+            // Send generated tokens and filter stopped entries
+            filter_send_generations(generations, entries);
+
+            // Filter next batch and remove requests that were stopped
+            let next_batch = filter_batch(client, next_batch, entries).await;
+
+            if let Some(concat_duration) = timings.concat {
+                metrics::histogram!("tgi_batch_concat_duration", "method" => "decode")
+                    .record(concat_duration.as_secs_f64());
+            }
+            metrics::histogram!("tgi_batch_forward_duration", "method" => "decode")
+                .record(timings.forward.as_secs_f64());
+            metrics::histogram!("tgi_batch_decode_duration", "method" => "decode")
+                .record(timings.decode.as_secs_f64());
+            metrics::histogram!("tgi_batch_filter_duration", "method" => "decode")
+                .record(start_filtering_time.elapsed().as_secs_f64());
+            metrics::histogram!("tgi_batch_inference_duration", "method" => "decode")
+                .record(start_time.elapsed().as_secs_f64());
+            metrics::counter!("tgi_batch_inference_success", "method" => "decode").increment(1);
+            next_batch
+        }
+        // If we have an error, we discard the whole batch
+        Err(err) => {
+            for id in batch_ids {
+                let _ = client.clear_cache(Some(id)).await;
+            }
+            send_errors(err, entries);
+            metrics::counter!("tgi_batch_inference_failure", "method" => "decode").increment(1);
+            None
+        }
+    }
+}
+
+/// Filter a `batch` and remove all requests not present in `entries`
+#[instrument(skip_all)]
+async fn filter_batch(
+    client: &mut ShardedClient,
+    next_batch: Option<CachedBatch>,
+    entries: &IntMap<u64, Entry>,
+) -> Option<CachedBatch> {
+    let mut batch = next_batch?;
+
+    // No need to filter
+    if batch.size as usize == entries.len() {
+        return Some(batch);
+    }
+
+    let id = batch.id;
+
+    // Retain only requests that are still in entries
+    batch.request_ids.retain(|id| entries.contains_key(id));
+
+    if batch.request_ids.is_empty() {
+        // All requests have been filtered out
+        // Next batch is now empty
+        // Clear it from the Python shards cache
+        // We unwrap here as we need to panic since we cannot recover if this method fails
+        client.clear_cache(Some(id)).await.unwrap();
+        None
+    } else {
+        // Filter Python shard cache
+        // We unwrap here as we need to panic since we cannot recover if this method fails
+        client.filter_batch(id, batch.request_ids).await.unwrap()
+    }
+}
+
+/// Send one or multiple `InferStreamResponse` to Infer for all `entries`
+/// and filter entries
+#[instrument(skip_all)]
+fn filter_send_generations(generations: Vec<Generation>, entries: &mut IntMap<u64, Entry>) {
+    generations.into_iter().for_each(|generation| {
+        let id = generation.request_id;
+        // Get entry
+        // We can `expect` here as the request id should always be in the entries
+        let entry = entries
+            .get(&id)
+            .expect("ID not found in entries. This is a bug.");
+
+        // Create and enter a span to link this function back to the entry
+        let _span = info_span!(parent: entry.temp_span.as_ref().expect("batch_span is None. This is a bug."), "send_generation", generation = ?generation).entered();
+        // Send generation responses back to the infer task
+        // If the receive an error from the Flume channel, it means that the client dropped the
+        // request and we need to stop generating hence why we unwrap_or(true)
+        let stopped = send_responses(generation, entry).inspect_err(|_err| {
+            tracing::error!("Entry response channel error.");
+            metrics::counter!("tgi_request_failure", "err" => "dropped").increment(1);
+        }).unwrap_or(true);
+        if stopped {
+            entries.remove(&id).expect("ID not found in entries. This is a bug.");
+        }
+    });
+}
+
+/// Send responses through the `entry` response channel
+fn send_responses(
+    generation: Generation,
+    entry: &Entry,
+) -> Result<bool, Box<SendError<Result<InferStreamResponse, InferError>>>> {
+    // Return directly if the channel is disconnected
+    if entry.response_tx.is_closed() {
+        metrics::counter!("tgi_request_failure", "err" => "dropped").increment(1);
+        return Ok(true);
+    }
+
+    let mut stopped = false;
+
+    if let Some(prefill_tokens) = generation.prefill_tokens {
+        // Create Token objects
+        // We do that here instead of in the Python code as Rust for loops are faster
+        let prefill_tokens = prefill_tokens
+            .ids
+            .into_iter()
+            .zip(prefill_tokens.logprobs)
+            .zip(prefill_tokens.texts)
+            .map(|((id, logprob), text)| PrefillToken { id, text, logprob })
+            .collect();
+
+        // Send message
+        entry
+            .response_tx
+            .send(Ok(InferStreamResponse::Prefill(prefill_tokens)))?;
+    }
+
+    // Create last Token
+    let tokens_ = generation.tokens.expect("Non empty tokens in generation");
+    let n = tokens_.ids.len();
+    metrics::histogram!("tgi_request_skipped_tokens").record((n - 1) as f64);
+    let mut iterator = tokens_
+        .ids
+        .into_iter()
+        .zip(tokens_.logprobs)
+        .zip(tokens_.texts)
+        .zip(tokens_.is_special)
+        .enumerate()
+        .peekable();
+    while let Some((i, (((id, logprob), text), special))) = iterator.next() {
+        let token = Token {
+            id,
+            text,
+            logprob,
+            special,
+        };
+        let top_tokens = if let Some(top_tokens_) = generation.top_tokens.get(i) {
+            top_tokens_
+                .ids
+                .iter()
+                .zip(top_tokens_.logprobs.iter())
+                .zip(top_tokens_.texts.iter())
+                .zip(top_tokens_.is_special.iter())
+                .map(|(((&id, &logprob), text), &special)| Token {
+                    id,
+                    text: text.to_string(),
+                    logprob,
+                    special,
+                })
+                .collect()
+        } else {
+            vec![]
+        };
+        match (&generation.generated_text, iterator.peek()) {
+            (Some(generated_text), None) => {
+                // Generation has ended
+                stopped = true;
+                // Send message
+                entry.response_tx.send(Ok(InferStreamResponse::End {
+                    token,
+                    top_tokens,
+                    generated_text: GeneratedText::from(generated_text.clone()),
+                    queued: entry.queue_time,
+                    start: entry.batch_time.unwrap(),
+                }))?;
+            }
+            _ => {
+                // Send message
+                entry
+                    .response_tx
+                    .send(Ok(InferStreamResponse::Intermediate { token, top_tokens }))?;
+            }
+        }
+    }
+
+    Ok(stopped)
+}
+
+/// Send errors to Infer for all `entries`
+#[instrument(skip_all)]
+fn send_errors(error: ClientError, entries: &mut IntMap<u64, Entry>) {
+    entries.drain().for_each(|(_, entry)| {
+        // Create and enter a span to link this function back to the entry
+        let _send_error_span = info_span!(parent: entry.temp_span.as_ref().expect("batch_span is None. This is a bug."), "send_error").entered();
+        let err = InferError::GenerationError(error.to_string());
+        metrics::counter!("tgi_request_failure", "err" => "generation").increment(1);
+        tracing::error!("{err}");
+
+        // unwrap_or is valid here as we don't care if the receiver is gone.
+        entry
+            .response_tx
+            .send(Err(err))
+            .unwrap_or(());
+    });
+}
+
+impl From<crate::client::GeneratedText> for GeneratedText {
+    fn from(value: crate::client::GeneratedText) -> Self {
+        let v3_finish_reason = crate::client::FinishReason::try_from(value.finish_reason).unwrap();
+        let finish_reason = match v3_finish_reason {
+            crate::client::FinishReason::Length => FinishReason::Length,
+            crate::client::FinishReason::EosToken => FinishReason::EndOfSequenceToken,
+            crate::client::FinishReason::StopSequence => FinishReason::StopSequence,
+        };
+
+        Self {
+            text: value.text,
+            generated_tokens: value.generated_tokens,
+            finish_reason,
+            seed: value.seed,
+        }
+    }
+}
diff --git a/backends/v3/src/block_allocator.rs b/backends/v3/src/block_allocator.rs
new file mode 100644
index 00000000..4fea172b
--- /dev/null
+++ b/backends/v3/src/block_allocator.rs
@@ -0,0 +1,209 @@
+use std::sync::Arc;
+use tokio::sync::{mpsc, oneshot};
+
+use crate::radix::RadixAllocator;
+
+#[derive(Debug, Clone)]
+pub struct BlockAllocation {
+    pub allocation_id: u64,
+    pub blocks: Vec<u32>,
+    pub slots: Vec<u32>,
+
+    /// Prefix that was cached and for which the KV does not have to
+    /// be recomputed.
+    pub prefix_len: u32,
+
+    pub(crate) block_allocator: Option<BlockAllocator>,
+}
+
+impl Drop for BlockAllocation {
+    fn drop(&mut self) {
+        if let Some(block_allocator) = self.block_allocator.as_mut() {
+            block_allocator.free(self.blocks.clone(), self.allocation_id)
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct BlockAllocator {
+    /// Channel to communicate with the background task
+    block_allocator: mpsc::UnboundedSender<BlockAllocatorCommand>,
+}
+
+impl BlockAllocator {
+    pub(crate) fn new(
+        max_batch_total_tokens: u32,
+        block_size: u32,
+        prefix_caching: bool,
+        window_size: Option<u32>,
+    ) -> Self {
+        // Create channel
+        let (sender, receiver) = mpsc::unbounded_channel();
+
+        // Launch background queue task
+        tokio::spawn(block_allocator_task(
+            max_batch_total_tokens / block_size,
+            block_size,
+            prefix_caching,
+            window_size,
+            receiver,
+        ));
+
+        Self {
+            block_allocator: sender,
+        }
+    }
+
+    pub(crate) async fn allocate(
+        &self,
+        tokens: u32,
+        prefill_tokens: Option<Arc<Vec<u32>>>,
+    ) -> Option<BlockAllocation> {
+        let (response_sender, response_receiver) = oneshot::channel();
+        self.block_allocator
+            .send(BlockAllocatorCommand::Allocate {
+                tokens,
+                prefill_tokens,
+                response_sender,
+            })
+            .unwrap();
+
+        response_receiver.await.unwrap().map(|mut allocation| {
+            allocation.block_allocator = Some(self.clone());
+            allocation
+        })
+    }
+
+    pub(crate) fn free(&self, blocks: Vec<u32>, allocation_id: u64) {
+        self.block_allocator
+            .send(BlockAllocatorCommand::Free {
+                allocation_id,
+                blocks,
+            })
+            .unwrap();
+    }
+}
+
+async fn block_allocator_task(
+    blocks: u32,
+    block_size: u32,
+    prefix_caching: bool,
+    window_size: Option<u32>,
+    mut receiver: mpsc::UnboundedReceiver<BlockAllocatorCommand>,
+) {
+    let mut allocator: Box<dyn Allocator + Send> = if prefix_caching {
+        Box::new(RadixAllocator::new(block_size, blocks, window_size))
+    } else {
+        Box::new(SimpleAllocator::new(blocks, block_size, window_size))
+    };
+    while let Some(cmd) = receiver.recv().await {
+        match cmd {
+            BlockAllocatorCommand::Free {
+                blocks,
+                allocation_id,
+            } => allocator.free(blocks, allocation_id),
+            BlockAllocatorCommand::Allocate {
+                tokens,
+                prefill_tokens,
+                response_sender,
+            } => {
+                response_sender
+                    .send(allocator.allocate(tokens, prefill_tokens))
+                    .unwrap();
+            }
+        }
+    }
+}
+
+#[derive(Debug)]
+enum BlockAllocatorCommand {
+    Free {
+        blocks: Vec<u32>,
+        allocation_id: u64,
+    },
+    Allocate {
+        tokens: u32,
+        prefill_tokens: Option<Arc<Vec<u32>>>,
+        response_sender: oneshot::Sender<Option<BlockAllocation>>,
+    },
+}
+
+pub trait Allocator {
+    fn allocate(
+        &mut self,
+        tokens: u32,
+        prefill_tokens: Option<Arc<Vec<u32>>>,
+    ) -> Option<BlockAllocation>;
+
+    fn free(&mut self, blocks: Vec<u32>, allocation_id: u64);
+}
+pub struct SimpleAllocator {
+    free_blocks: Vec<u32>,
+    block_size: u32,
+    window_size: Option<u32>,
+}
+
+impl SimpleAllocator {
+    fn new(blocks: u32, block_size: u32, window_size: Option<u32>) -> Self {
+        SimpleAllocator {
+            block_size,
+            // Block 0 is reserved for health checks
+            free_blocks: (1..blocks).collect(),
+            window_size,
+        }
+    }
+}
+
+impl Allocator for SimpleAllocator {
+    fn allocate(
+        &mut self,
+        tokens: u32,
+        _prefill_tokens: Option<Arc<Vec<u32>>>,
+    ) -> Option<BlockAllocation> {
+        // Apply window size
+        let (required_blocks, repeats) = {
+            let (tokens, repeats) = match self.window_size {
+                None => (tokens, 1),
+                Some(window_size) => {
+                    let repeats = (tokens + window_size - 1) / window_size;
+                    let tokens = core::cmp::min(tokens, window_size);
+                    (tokens, repeats as usize)
+                }
+            };
+            // Pad to a multiple of block size
+            let required_blocks = (tokens + self.block_size - 1) / self.block_size;
+            (required_blocks, repeats)
+        };
+
+        let tokens = tokens as usize;
+        if required_blocks > self.free_blocks.len() as u32 {
+            None
+        } else {
+            let blocks = self
+                .free_blocks
+                .split_off(self.free_blocks.len() - required_blocks as usize);
+            let mut slots =
+                Vec::with_capacity((required_blocks * self.block_size * repeats as u32) as usize);
+
+            'slots: for block_id in blocks.repeat(repeats).iter() {
+                for s in (block_id * self.block_size)..((block_id + 1) * self.block_size) {
+                    slots.push(s);
+                    if slots.len() == tokens {
+                        break 'slots;
+                    }
+                }
+            }
+            Some(BlockAllocation {
+                allocation_id: 0,
+                blocks,
+                slots,
+                prefix_len: 0,
+                block_allocator: None,
+            })
+        }
+    }
+
+    fn free(&mut self, blocks: Vec<u32>, _allocation_id: u64) {
+        self.free_blocks.extend(blocks)
+    }
+}
diff --git a/backends/v3/src/client/grpc_client.rs b/backends/v3/src/client/grpc_client.rs
new file mode 100644
index 00000000..4508b92d
--- /dev/null
+++ b/backends/v3/src/client/grpc_client.rs
@@ -0,0 +1,288 @@
+/// Single shard Client
+use crate::client::{pb, Chunk};
+use crate::client::{ClientError, Result, WARMUP_IMAGE_BASE64};
+use base64::engine::general_purpose::STANDARD;
+use base64::Engine;
+use grpc_metadata::InjectTelemetryContext;
+use pb::generate::v3::text_generation_service_client::TextGenerationServiceClient;
+use pb::generate::v3::*;
+use std::cmp::min;
+use std::time::Duration;
+use tonic::transport::{Channel, Uri};
+use tracing::instrument;
+
+/// Text Generation Inference gRPC client
+#[derive(Debug, Clone)]
+pub struct Client {
+    stub: TextGenerationServiceClient<Channel>,
+}
+
+impl Client {
+    /// Returns a client connected to the given url
+    #[allow(dead_code)]
+    pub async fn connect(uri: Uri) -> Result<Self> {
+        let channel = Channel::builder(uri).connect().await?;
+
+        Ok(Self {
+            stub: TextGenerationServiceClient::new(channel),
+        })
+    }
+
+    /// Returns a client connected to the given unix socket
+    pub async fn connect_uds(path: String) -> Result<Self> {
+        let channel = Channel::from_shared("http://[::]:50051".to_string())
+            .unwrap()
+            .connect_with_connector(tower::service_fn(move |_: Uri| {
+                tokio::net::UnixStream::connect(path.clone())
+            }))
+            .await?;
+
+        Ok(Self {
+            stub: TextGenerationServiceClient::new(channel),
+        })
+    }
+
+    /// Returns a list of uris or unix sockets of all shards
+    #[instrument(skip(self))]
+    pub async fn service_discovery(&mut self) -> Result<Vec<String>> {
+        let request = tonic::Request::new(ServiceDiscoveryRequest {}).inject_context();
+        let response = self.stub.service_discovery(request).await.map_err(|_| {
+            ClientError::Connection("Server does not support v3 interface".to_string())
+        })?;
+        let urls = response
+            .into_inner()
+            .urls
+            .into_iter()
+            // Remove unix socket prefix
+            .map(|url| match url.strip_prefix("unix://") {
+                None => url,
+                Some(stripped_url) => stripped_url.to_string(),
+            })
+            .collect();
+        Ok(urls)
+    }
+
+    /// Get model info
+    #[instrument(skip(self))]
+    pub async fn info(&mut self) -> Result<InfoResponse> {
+        let request = tonic::Request::new(InfoRequest {}).inject_context();
+        let response = self.stub.info(request).await?.into_inner();
+        Ok(response)
+    }
+
+    /// Get model health
+    #[instrument(skip(self))]
+    pub async fn health(&mut self) -> Result<HealthResponse> {
+        let request = tonic::Request::new(HealthRequest {}).inject_context();
+        let response = self.stub.health(request).await?.into_inner();
+        Ok(response)
+    }
+
+    /// Clear the past generations cache
+    #[instrument(skip(self))]
+    pub async fn clear_cache(&mut self, batch_id: Option<u64>) -> Result<()> {
+        let request = tonic::Request::new(ClearCacheRequest { id: batch_id }).inject_context();
+        self.stub.clear_cache(request).await?;
+        Ok(())
+    }
+
+    /// Filter a cached batch
+    #[instrument(skip(self))]
+    pub async fn filter_batch(
+        &mut self,
+        batch_id: u64,
+        request_ids: Vec<u64>,
+    ) -> Result<Option<CachedBatch>> {
+        let request = tonic::Request::new(FilterBatchRequest {
+            batch_id,
+            request_ids,
+        })
+        .inject_context();
+        let filtered_batch = self.stub.filter_batch(request).await?.into_inner();
+        Ok(filtered_batch.batch)
+    }
+
+    /// Warmup on a max size batch
+    ///
+    /// Returns the maximum amount of tokens supported by the hardware
+    #[instrument(skip_all)]
+    pub async fn warmup(
+        &mut self,
+        max_input_length: u32,
+        max_prefill_tokens: u32,
+        max_total_tokens: u32,
+        max_batch_total_tokens: u32,
+        max_batch_size: Option<usize>,
+    ) -> Result<Option<u32>> {
+        let mut n_tokens = 0;
+        let mut requests = Vec::new();
+        // Create requests
+        while n_tokens < max_prefill_tokens {
+            let truncate = min(max_input_length, max_prefill_tokens - n_tokens);
+
+            let mut input_chunks = Vec::new();
+            input_chunks
+                .push(Chunk::Text("_test ".to_string().repeat(max_input_length as usize)).into());
+            if n_tokens == 0 {
+                input_chunks.push(
+                    Chunk::Image(Image {
+                        // Safe unwrap, because we control the data.
+                        data: STANDARD.decode(WARMUP_IMAGE_BASE64).unwrap(),
+                        mimetype: "image/jpeg;base64".to_string(),
+                    })
+                    .into(),
+                );
+            }
+
+            // Send stringly-typed inputs for compatibility for backends that haven't
+            // been updated to support chunks.
+
+            let mut inputs = String::new();
+            inputs.push_str(&"_test ".to_string().repeat(max_input_length as usize));
+            if n_tokens == 0 {
+                // 1 request is enough to test vision heads.
+                // Sending images on other queries messes up easily with truncation.
+                inputs.push_str(&format!(
+                    "![](data:image/jpeg;base64,{WARMUP_IMAGE_BASE64})",
+                ));
+            }
+
+            requests.push(Request {
+                id: 0,
+                inputs,
+                add_special_tokens: true,
+                input_chunks: Some(Input {
+                    chunks: input_chunks,
+                }),
+                // We truncate the input on the server side to be sure that it has the correct size
+                truncate,
+                // Blocks and slots will be set on the server side if we use paged attention
+                blocks: vec![],
+                slots: vec![],
+                prefix_len: 0,
+                // Set sampling parameters to also take these ops into account in the max memory
+                parameters: Some(NextTokenChooserParameters {
+                    temperature: 0.9,
+                    top_k: 10,
+                    top_p: 0.9,
+                    typical_p: 0.9,
+                    do_sample: false,
+                    seed: 0,
+                    repetition_penalty: 1.2,
+                    frequency_penalty: 0.1,
+                    watermark: true,
+                    grammar: String::new(),
+                    grammar_type: GrammarType::None as i32,
+                }),
+                stopping_parameters: Some(StoppingCriteriaParameters {
+                    max_new_tokens: max_total_tokens - truncate,
+                    stop_sequences: vec![],
+                    ignore_eos_token: true,
+                }),
+                prefill_logprobs: true,
+                top_n_tokens: 20,
+                adapter_id: None,
+            });
+            n_tokens += max_input_length;
+
+            // Check max_batch_size
+            if Some(requests.len()) == max_batch_size {
+                break;
+            }
+        }
+
+        let batch = Batch {
+            id: 0,
+            size: requests.len() as u32,
+            requests,
+            max_tokens: max_input_length,
+            max_blocks: 0,
+        };
+
+        let request = tonic::Request::new(WarmupRequest {
+            batch: Some(batch),
+            max_input_length,
+            max_prefill_tokens,
+            max_total_tokens,
+            max_batch_total_tokens,
+        })
+        .inject_context();
+        let response = self.stub.warmup(request).await?.into_inner();
+        Ok(response.max_supported_total_tokens)
+    }
+
+    /// Generate one token for each request in the given batch
+    ///
+    /// Returns Generation for each request in batch
+    /// and the next cached batch
+    #[instrument(skip_all, fields(id = &batch.id, size = &batch.size))]
+    pub async fn prefill(
+        &mut self,
+        batch: Batch,
+    ) -> Result<(Vec<Generation>, Option<CachedBatch>, PrefillTimings)> {
+        let request = tonic::Request::new(PrefillRequest { batch: Some(batch) }).inject_context();
+        let response = self.stub.prefill(request).await?.into_inner();
+        Ok((
+            response.generations,
+            response.batch,
+            PrefillTimings::new(response.forward_ns, response.decode_ns, response.total_ns),
+        ))
+    }
+
+    /// Generate one token for each request in the given cached batches
+    ///
+    /// Returns Generation for each request in batches
+    /// and the next cached batch
+    #[instrument(skip_all, fields(size = batches.iter().map(|batch|{batch.size}).sum::<u32>()))]
+    pub async fn decode(
+        &mut self,
+        batches: Vec<CachedBatch>,
+    ) -> Result<(Vec<Generation>, Option<CachedBatch>, DecodeTimings)> {
+        let request = tonic::Request::new(DecodeRequest { batches }).inject_context();
+        let response = self.stub.decode(request).await?.into_inner();
+        Ok((
+            response.generations,
+            response.batch,
+            DecodeTimings::new(
+                response.concat_ns,
+                response.forward_ns,
+                response.decode_ns,
+                response.total_ns,
+            ),
+        ))
+    }
+}
+
+pub struct PrefillTimings {
+    pub forward: Duration,
+    pub decode: Duration,
+    pub total: Duration,
+}
+
+impl PrefillTimings {
+    fn new(forward_ns: u64, decode_ns: u64, total_ns: u64) -> Self {
+        Self {
+            forward: Duration::from_nanos(forward_ns),
+            decode: Duration::from_nanos(decode_ns),
+            total: Duration::from_nanos(total_ns),
+        }
+    }
+}
+
+pub struct DecodeTimings {
+    pub concat: Option<Duration>,
+    pub forward: Duration,
+    pub decode: Duration,
+    pub total: Duration,
+}
+
+impl DecodeTimings {
+    fn new(concat_ns: Option<u64>, forward_ns: u64, decode_ns: u64, total_ns: u64) -> Self {
+        Self {
+            concat: concat_ns.map(Duration::from_nanos),
+            forward: Duration::from_nanos(forward_ns),
+            decode: Duration::from_nanos(decode_ns),
+            total: Duration::from_nanos(total_ns),
+        }
+    }
+}
diff --git a/backends/v3/src/client/mod.rs b/backends/v3/src/client/mod.rs
new file mode 100644
index 00000000..755431f4
--- /dev/null
+++ b/backends/v3/src/client/mod.rs
@@ -0,0 +1,76 @@
+//! Text Generation gRPC client library
+
+use async_trait::async_trait;
+use thiserror::Error;
+use tonic::transport;
+use tonic::Status;
+
+#[allow(clippy::derive_partial_eq_without_eq)]
+mod pb;
+
+mod grpc_client;
+mod sharded_client;
+
+pub use grpc_client::Client;
+pub use pb::generate::v3::{
+    input_chunk::Chunk, Batch, CachedBatch, FinishReason, GeneratedText, Generation, GrammarType,
+    HealthResponse, Image, InfoResponse, Input, InputChunk, NextTokenChooserParameters, Request,
+    StoppingCriteriaParameters,
+};
+pub use sharded_client::ShardedClient;
+
+#[async_trait]
+pub trait Health {
+    /// Check if a generate server is healthy by asking it to allocate a tensor on device
+    async fn device_health(&self) -> Result<()>;
+
+    /// Check if a generate server is healthy by doing a forward pass.
+    /// EXPENSIVE
+    async fn model_health(&self) -> Result<()>;
+}
+
+#[derive(Debug)]
+pub struct ShardInfo {
+    pub requires_padding: bool,
+    pub dtype: String,
+    pub device_type: String,
+    pub window_size: Option<u32>,
+    pub speculate: u32,
+}
+
+#[derive(Error, Debug, Clone)]
+pub enum ClientError {
+    #[error("Could not connect to Text Generation server: {0}")]
+    Connection(String),
+    #[error("Server error: {0}")]
+    Generation(String),
+    #[error("Sharded results are empty")]
+    EmptyResults,
+}
+
+impl From<Status> for ClientError {
+    fn from(err: Status) -> Self {
+        let err = Self::Generation(err.message().to_string());
+        tracing::error!("{err}");
+        err
+    }
+}
+
+impl From<transport::Error> for ClientError {
+    fn from(err: transport::Error) -> Self {
+        let err = Self::Connection(err.to_string());
+        tracing::error!("{err}");
+        err
+    }
+}
+
+// Small convenience re-wrapping of `Chunk`.
+impl From<Chunk> for InputChunk {
+    fn from(chunk: Chunk) -> Self {
+        InputChunk { chunk: Some(chunk) }
+    }
+}
+
+static WARMUP_IMAGE_BASE64 :&str = "iVBORw0KGgoAAAANSUhEUgAAABQAAAAUCAIAAAAC64paAAABg2lDQ1BJQ0MgcHJvZmlsZQAAKJF9kT1Iw0AcxV/TSotUROxQxCFDdbKLijjWKhShQqgVWnUwufQLmrQkKS6OgmvBwY/FqoOLs64OroIg+AHi7OCk6CIl/i8ptIjx4Lgf7+497t4BQqvKNDOQADTdMjKppJjLr4rBVwQQwhAERGVm1uckKQ3P8XUPH1/v4jzL+9yfY0AtmAzwicQJVjcs4g3imU2rznmfOMLKskp8Tjxh0AWJH7muuPzGueSwwDMjRjYzTxwhFks9rPQwKxsa8TRxTNV0yhdyLquctzhr1Qbr3JO/MFzQV5a5TnMUKSxiCRJEKGiggiosxGnVSTGRof2kh3/E8UvkUshVASPHAmrQIDt+8D/43a1ZnJp0k8JJoO/Ftj/GgOAu0G7a9vexbbdPAP8zcKV3/bUWMPtJerOrxY6AwW3g4rqrKXvA5Q4QfarLhuxIfppCsQi8n9E35YHhW6B/ze2ts4/TByBLXaVvgINDYLxE2ese7w719vbvmU5/PycecohsjayNAAAACXBIWXMAAC4jAAAuIwF4pT92AAAAB3RJTUUH6AQIEQMnlTSSjwAAABl0RVh0Q29tbWVudABDcmVhdGVkIHdpdGggR0lNUFeBDhcAAAASSURBVDjLY2AYBaNgFIyCoQsABMQAAeRw1DoAAAAASUVORK5CYII=";
+
+pub type Result<T> = std::result::Result<T, ClientError>;
diff --git a/backends/v3/src/client/sharded_client.rs b/backends/v3/src/client/sharded_client.rs
new file mode 100644
index 00000000..0b6e916e
--- /dev/null
+++ b/backends/v3/src/client/sharded_client.rs
@@ -0,0 +1,264 @@
+use crate::client::{ClientError, Result};
+/// Multi shard Client
+use crate::client::{Health, ShardInfo};
+
+use crate::client::grpc_client::{DecodeTimings, PrefillTimings};
+use crate::client::{
+    Batch, CachedBatch, Client, Generation, GrammarType, HealthResponse,
+    NextTokenChooserParameters, Request, StoppingCriteriaParameters,
+};
+use crate::client::{Chunk, InfoResponse, Input};
+use async_trait::async_trait;
+use futures::future::join_all;
+use tonic::transport::Uri;
+use tracing::instrument;
+
+#[derive(Debug, Clone)]
+/// Text Generation Inference gRPC multi client
+pub struct ShardedClient {
+    clients: Vec<Client>,
+}
+
+impl ShardedClient {
+    fn new(clients: Vec<Client>) -> Self {
+        Self { clients }
+    }
+
+    /// Create a new ShardedClient from a master client. The master client will communicate with
+    /// the other shards and returns all uris/unix sockets with the `service_discovery` gRPC method.
+    async fn from_master_client(mut master_client: Client) -> Result<Self> {
+        // Get all uris/unix sockets from the master client
+        let uris = master_client.service_discovery().await?;
+        let futures = uris.into_iter().map(Client::connect_uds);
+        let clients: Result<Vec<Client>> = join_all(futures).await.into_iter().collect();
+        Ok(Self::new(clients?))
+    }
+
+    /// Returns a client connected to the given uri
+    #[allow(dead_code)]
+    pub async fn connect(uri: Uri) -> Result<Self> {
+        let master_client = Client::connect(uri).await?;
+        Self::from_master_client(master_client).await
+    }
+
+    /// Returns a client connected to the given unix socket
+    pub async fn connect_uds(path: String) -> Result<Self> {
+        let master_client = Client::connect_uds(path).await?;
+        Self::from_master_client(master_client).await
+    }
+
+    /// Get the model info
+    #[instrument(skip(self))]
+    pub async fn info(&mut self) -> Result<ShardInfo> {
+        let futures: Vec<_> = self
+            .clients
+            .iter_mut()
+            .map(|client| client.info())
+            .collect();
+        join_all(futures).await.pop().unwrap().map(ShardInfo::from)
+    }
+
+    /// GRPC health check
+    #[instrument(skip(self))]
+    pub async fn health(&mut self) -> Result<HealthResponse> {
+        let futures: Vec<_> = self
+            .clients
+            .iter_mut()
+            .map(|client| client.health())
+            .collect();
+        join_all(futures).await.pop().unwrap()
+    }
+
+    /// Clear the past generations cache
+    #[instrument(skip(self))]
+    pub async fn clear_cache(&mut self, batch_id: Option<u64>) -> Result<()> {
+        let futures: Vec<_> = self
+            .clients
+            .iter_mut()
+            .map(|client| client.clear_cache(batch_id))
+            .collect();
+        join_all(futures).await.into_iter().collect()
+    }
+
+    /// Filter a cached batch
+    #[instrument(skip(self))]
+    pub async fn filter_batch(
+        &mut self,
+        batch_id: u64,
+        request_ids: Vec<u64>,
+    ) -> Result<Option<CachedBatch>> {
+        let futures: Vec<_> = self
+            .clients
+            .iter_mut()
+            .map(|client| Box::pin(client.filter_batch(batch_id, request_ids.clone())))
+            .collect();
+        // all shards return the same message
+        join_all(futures).await.pop().unwrap()
+    }
+
+    /// Warmup on a max size batch
+    ///
+    /// Returns the maximum amount of tokens supported by the hardware
+    #[instrument(skip(self))]
+    pub async fn warmup(
+        &mut self,
+        max_input_length: u32,
+        max_prefill_tokens: u32,
+        max_total_tokens: u32,
+        max_batch_total_tokens: u32,
+        max_batch_size: Option<usize>,
+    ) -> Result<Option<u32>> {
+        let futures: Vec<_> = self
+            .clients
+            .iter_mut()
+            .map(|client| {
+                Box::pin(client.warmup(
+                    max_input_length,
+                    max_prefill_tokens,
+                    max_total_tokens,
+                    max_batch_total_tokens,
+                    max_batch_size,
+                ))
+            })
+            .collect();
+        // Take the minimum value
+        let results = join_all(futures)
+            .await
+            .into_iter()
+            .collect::<Result<Vec<Option<u32>>>>()?;
+        Ok(results.into_iter().flatten().min())
+    }
+
+    /// Generate one token for each request in the given batch
+    ///
+    /// Returns Generation for each request in batch
+    /// and the next cached batch
+    #[instrument(skip_all, fields(id = & batch.id, size = & batch.size))]
+    pub async fn prefill(
+        &mut self,
+        batch: Batch,
+    ) -> Result<(Vec<Generation>, Option<CachedBatch>, PrefillTimings)> {
+        let futures: Vec<_> = self
+            .clients
+            .iter_mut()
+            .map(|client| Box::pin(client.prefill(batch.clone())))
+            .collect();
+        #[allow(clippy::type_complexity)]
+        let results: Result<Vec<(Vec<Generation>, Option<CachedBatch>, PrefillTimings)>> =
+            join_all(futures).await.into_iter().collect();
+        let mut results = results?;
+
+        let (mut generations, next_batch, mut timings) =
+            results.pop().ok_or(ClientError::EmptyResults)?;
+
+        // Merge generations from different model shards
+        for (mut shard_generations, _, shard_timings) in results.into_iter() {
+            generations.append(&mut shard_generations);
+            // Return the timings of the slowest shard
+            if shard_timings.total > timings.total {
+                timings = shard_timings;
+            }
+        }
+        Ok((generations, next_batch, timings))
+    }
+
+    /// Generate one token for each request in the given cached batches
+    ///
+    /// Returns Generation for each request in batches
+    /// and the next cached batch
+    #[instrument(skip_all, fields(size = batches.iter().map(| batch | {batch.size}).sum::< u32 > ()))]
+    pub async fn decode(
+        &mut self,
+        batches: Vec<CachedBatch>,
+    ) -> Result<(Vec<Generation>, Option<CachedBatch>, DecodeTimings)> {
+        let futures: Vec<_> = self
+            .clients
+            .iter_mut()
+            .map(|client| Box::pin(client.decode(batches.clone())))
+            .collect();
+        #[allow(clippy::type_complexity)]
+        let results: Result<Vec<(Vec<Generation>, Option<CachedBatch>, DecodeTimings)>> =
+            join_all(futures).await.into_iter().collect();
+        let mut results = results?;
+
+        let (mut generations, next_batch, mut timings) =
+            results.pop().ok_or(ClientError::EmptyResults)?;
+
+        // Merge generations from different model shards
+        for (mut shard_generations, _, shard_timings) in results.into_iter() {
+            generations.append(&mut shard_generations);
+            // Return the timings of the slowest shard
+            if shard_timings.total > timings.total {
+                timings = shard_timings;
+            }
+        }
+        Ok((generations, next_batch, timings))
+    }
+}
+
+impl From<InfoResponse> for ShardInfo {
+    fn from(value: InfoResponse) -> Self {
+        Self {
+            requires_padding: value.requires_padding,
+            dtype: value.dtype,
+            device_type: value.device_type,
+            window_size: value.window_size,
+            speculate: value.speculate,
+        }
+    }
+}
+
+#[async_trait]
+impl Health for ShardedClient {
+    async fn device_health(&self) -> Result<()> {
+        self.clone().health().await?;
+        Ok(())
+    }
+
+    async fn model_health(&self) -> Result<()> {
+        // Dummy batch of 1 token and 1 generated token
+        let liveness_request = Request {
+            id: u64::MAX,
+            inputs: "liveness".to_string(),
+            input_chunks: Some(Input {
+                chunks: vec![Chunk::Text("liveness".into()).into()],
+            }),
+            truncate: 10,
+            add_special_tokens: true,
+            prefill_logprobs: false,
+            parameters: Some(NextTokenChooserParameters {
+                temperature: 1.0,
+                top_k: 0,
+                top_p: 1.0,
+                typical_p: 1.0,
+                do_sample: false,
+                seed: 0,
+                repetition_penalty: 1.0,
+                frequency_penalty: 0.0,
+                watermark: false,
+                grammar: String::new(),
+                grammar_type: GrammarType::None as i32,
+            }),
+            stopping_parameters: Some(StoppingCriteriaParameters {
+                max_new_tokens: 1,
+                stop_sequences: vec![],
+                ignore_eos_token: false,
+            }),
+            top_n_tokens: 0,
+            // Block 0 is reserved for health checks
+            blocks: vec![0],
+            slots: (0..16).collect(),
+            prefix_len: 0,
+            adapter_id: None,
+        };
+        let batch = Batch {
+            id: u64::MAX,
+            requests: vec![liveness_request],
+            size: 1,
+            max_tokens: 2,
+            max_blocks: 1,
+        };
+        self.clone().prefill(batch).await?;
+        Ok(())
+    }
+}
diff --git a/backends/v3/src/lib.rs b/backends/v3/src/lib.rs
new file mode 100644
index 00000000..8913e40b
--- /dev/null
+++ b/backends/v3/src/lib.rs
@@ -0,0 +1,147 @@
+mod backend;
+pub mod block_allocator;
+mod client;
+mod queue;
+pub mod radix;
+
+use crate::client::{ClientError, ShardedClient};
+pub(crate) use backend::BackendV3;
+use serde::Serialize;
+use thiserror::Error;
+use utoipa::ToSchema;
+
+#[derive(Clone, Debug, Serialize, ToSchema)]
+pub struct BackendInfo {
+    /// Mandatory
+    #[schema(example = "cuda")]
+    pub model_device_type: String,
+    #[schema(example = "torch.float16")]
+    pub model_dtype: String,
+
+    /// Backend parameters
+    #[schema(example = "1")]
+    pub speculate: usize,
+    #[schema(example = "1.2")]
+    pub waiting_served_ratio: f32,
+    #[schema(example = "32000")]
+    pub max_batch_total_tokens: u32,
+    #[schema(example = "20")]
+    pub max_waiting_tokens: usize,
+    #[schema(nullable = true, example = "null")]
+    pub max_batch_size: Option<usize>,
+}
+
+#[allow(clippy::too_many_arguments)]
+pub async fn connect_backend(
+    max_input_tokens: usize,
+    max_total_tokens: usize,
+    master_shard_uds_path: String,
+    waiting_served_ratio: f32,
+    max_batch_prefill_tokens: u32,
+    max_batch_total_tokens: Option<u32>,
+    max_waiting_tokens: usize,
+    max_batch_size: Option<usize>,
+) -> Result<(BackendV3, BackendInfo), V3Error> {
+    // Helper function
+    let check_max_batch_total_tokens = |max_supported_batch_total_tokens: Option<u32>| {
+        match max_supported_batch_total_tokens {
+            // Older models do not support automatic max-batch-total-tokens
+            None => {
+                let max_batch_total_tokens = max_batch_total_tokens
+                    .unwrap_or(16000.max((max_total_tokens as u32).max(max_batch_prefill_tokens)));
+                tracing::warn!("Model does not support automatic max batch total tokens");
+                Ok(max_batch_total_tokens)
+            }
+            // Flash attention models return their max supported total tokens
+            Some(max_supported_batch_total_tokens) => {
+                // Warn if user added his own max-batch-total-tokens as we will ignore it
+                if max_batch_total_tokens.is_some() {
+                    tracing::warn!(
+                        "`--max-batch-total-tokens` is deprecated for Flash \
+                        Attention models."
+                    );
+                    tracing::warn!(
+                        "Inferred max batch total tokens: {max_supported_batch_total_tokens}"
+                    );
+                }
+                if max_total_tokens as u32 > max_supported_batch_total_tokens {
+                    return Err(V3Error::NotEnoughMemory(max_total_tokens));
+                }
+
+                Ok(max_supported_batch_total_tokens)
+            }
+        }
+    };
+
+    let mut sharded_client = ShardedClient::connect_uds(master_shard_uds_path)
+        .await
+        .map_err(V3Error::Connection)?;
+
+    // server is running on v3
+    // Clear the cache; useful if the webserver rebooted
+    sharded_client
+        .clear_cache(None)
+        .await
+        .map_err(V3Error::Cache)?;
+    // Get info from the shard
+    let shard_info = sharded_client.info().await.map_err(V3Error::Info)?;
+
+    // Warmup model
+    tracing::info!("Warming up model");
+    let max_batch_total_tokens = check_max_batch_total_tokens(
+        sharded_client
+            .warmup(
+                max_input_tokens as u32,
+                max_batch_prefill_tokens,
+                max_total_tokens as u32,
+                max_batch_total_tokens.unwrap_or(16000.max((max_total_tokens as u32).max(max_batch_prefill_tokens))),
+                max_batch_size,
+            )
+            .await
+            .map_err(V3Error::Warmup)?,
+    )?;
+    tracing::info!("Setting max batch total tokens to {max_batch_total_tokens}");
+    metrics::gauge!("tgi_batch_max_total_tokens").set(max_batch_total_tokens);
+
+    let backend_info = BackendInfo {
+        waiting_served_ratio,
+        max_batch_total_tokens,
+        max_waiting_tokens,
+        max_batch_size,
+        model_device_type: shard_info.device_type.clone(),
+        model_dtype: shard_info.dtype.clone(),
+        speculate: shard_info.speculate as usize,
+    };
+
+    let backend = BackendV3::new(
+        sharded_client,
+        waiting_served_ratio,
+        max_input_tokens as u32,
+        max_total_tokens as u32,
+        max_batch_prefill_tokens,
+        max_batch_total_tokens,
+        max_waiting_tokens,
+        max_batch_size,
+        shard_info.requires_padding,
+        shard_info.window_size,
+        shard_info.speculate,
+    );
+
+    tracing::info!("Using backend V3");
+
+    Ok((backend, backend_info))
+}
+
+#[derive(Debug, Error)]
+pub enum V3Error {
+    #[error("Unable to clear the Python model shards cache: {0}")]
+    Cache(ClientError),
+    #[error("Unable to connect to the Python model shards: {0}")]
+    Connection(ClientError),
+    #[error("Unable to get the Python model shards info: {0}")]
+    Info(ClientError),
+    #[error("Unable to warmup the Python model shards: {0}")]
+    Warmup(ClientError),
+    #[error("Not enough memory to handle `max_total_tokens={0}`")]
+    NotEnoughMemory(usize),
+}
diff --git a/backends/v3/src/main.rs b/backends/v3/src/main.rs
new file mode 100644
index 00000000..471ddb5a
--- /dev/null
+++ b/backends/v3/src/main.rs
@@ -0,0 +1,212 @@
+use clap::{Parser, Subcommand};
+use text_generation_router::{server, usage_stats};
+use text_generation_router_v3::{connect_backend, V3Error};
+use thiserror::Error;
+
+/// App Configuration
+#[derive(Parser, Debug)]
+#[clap(author, version, about, long_about = None)]
+struct Args {
+    #[command(subcommand)]
+    command: Option<Commands>,
+
+    #[clap(default_value = "128", long, env)]
+    max_concurrent_requests: usize,
+    #[clap(default_value = "2", long, env)]
+    max_best_of: usize,
+    #[clap(default_value = "4", long, env)]
+    max_stop_sequences: usize,
+    #[clap(default_value = "5", long, env)]
+    max_top_n_tokens: u32,
+    #[clap(default_value = "1024", long, env)]
+    max_input_tokens: usize,
+    #[clap(default_value = "2048", long, env)]
+    max_total_tokens: usize,
+    #[clap(default_value = "1.2", long, env)]
+    waiting_served_ratio: f32,
+    #[clap(default_value = "4096", long, env)]
+    max_batch_prefill_tokens: u32,
+    #[clap(long, env)]
+    max_batch_total_tokens: Option<u32>,
+    #[clap(default_value = "20", long, env)]
+    max_waiting_tokens: usize,
+    #[clap(long, env)]
+    max_batch_size: Option<usize>,
+    #[clap(default_value = "0.0.0.0", long, env)]
+    hostname: String,
+    #[clap(default_value = "3000", long, short, env)]
+    port: u16,
+    #[clap(default_value = "/tmp/text-generation-server-0", long, env)]
+    master_shard_uds_path: String,
+    #[clap(default_value = "bigscience/bloom", long, env)]
+    tokenizer_name: String,
+    #[clap(long, env)]
+    tokenizer_config_path: Option<String>,
+    #[clap(long, env)]
+    revision: Option<String>,
+    #[clap(default_value = "2", long, env)]
+    validation_workers: usize,
+    #[clap(long, env)]
+    api_key: Option<String>,
+    #[clap(long, env)]
+    json_output: bool,
+    #[clap(long, env)]
+    otlp_endpoint: Option<String>,
+    #[clap(default_value = "text-generation-inference.router", long, env)]
+    otlp_service_name: String,
+    #[clap(long, env)]
+    cors_allow_origin: Option<Vec<String>>,
+    #[clap(long, env)]
+    ngrok: bool,
+    #[clap(long, env)]
+    ngrok_authtoken: Option<String>,
+    #[clap(long, env)]
+    ngrok_edge: Option<String>,
+    #[clap(long, env, default_value_t = false)]
+    messages_api_enabled: bool,
+    #[clap(long, env, default_value_t = false)]
+    disable_grammar_support: bool,
+    #[clap(default_value = "4", long, env)]
+    max_client_batch_size: usize,
+    #[clap(default_value = "on", long, env)]
+    usage_stats: usage_stats::UsageStatsLevel,
+}
+
+#[derive(Debug, Subcommand)]
+enum Commands {
+    PrintSchema,
+}
+
+#[tokio::main]
+async fn main() -> Result<(), RouterError> {
+    // Get args
+    let args = Args::parse();
+    // Pattern match configuration
+    let Args {
+        command,
+        max_concurrent_requests,
+        max_best_of,
+        max_stop_sequences,
+        max_top_n_tokens,
+        max_input_tokens,
+        max_total_tokens,
+        waiting_served_ratio,
+        max_batch_prefill_tokens,
+        max_batch_total_tokens,
+        max_waiting_tokens,
+        max_batch_size,
+        hostname,
+        port,
+        master_shard_uds_path,
+        tokenizer_name,
+        tokenizer_config_path,
+        revision,
+        validation_workers,
+        api_key,
+        json_output,
+        otlp_endpoint,
+        otlp_service_name,
+        cors_allow_origin,
+        ngrok,
+        ngrok_authtoken,
+        ngrok_edge,
+        messages_api_enabled,
+        disable_grammar_support,
+        max_client_batch_size,
+        usage_stats,
+    } = args;
+
+    if let Some(Commands::PrintSchema) = command {
+        use utoipa::OpenApi;
+        let api_doc = text_generation_router::server::ApiDoc::openapi();
+        let api_doc = serde_json::to_string_pretty(&api_doc).unwrap();
+        println!("{}", api_doc);
+        std::process::exit(0);
+    };
+    text_generation_router::logging::init_logging(otlp_endpoint, otlp_service_name, json_output);
+
+    // Validate args
+    if max_input_tokens >= max_total_tokens {
+        return Err(RouterError::ArgumentValidation(
+            "`max_input_tokens` must be < `max_total_tokens`".to_string(),
+        ));
+    }
+    if max_input_tokens as u32 > max_batch_prefill_tokens {
+        return Err(RouterError::ArgumentValidation(format!("`max_batch_prefill_tokens` must be >= `max_input_tokens`. Given: {max_batch_prefill_tokens} and {max_input_tokens}")));
+    }
+
+    if validation_workers == 0 {
+        return Err(RouterError::ArgumentValidation(
+            "`validation_workers` must be > 0".to_string(),
+        ));
+    }
+
+    if let Some(ref max_batch_total_tokens) = max_batch_total_tokens {
+        if max_batch_prefill_tokens > *max_batch_total_tokens {
+            return Err(RouterError::ArgumentValidation(format!("`max_batch_prefill_tokens` must be <= `max_batch_total_tokens`. Given: {max_batch_prefill_tokens} and {max_batch_total_tokens}")));
+        }
+        if max_total_tokens as u32 > *max_batch_total_tokens {
+            return Err(RouterError::ArgumentValidation(format!("`max_total_tokens` must be <= `max_batch_total_tokens`. Given: {max_total_tokens} and {max_batch_total_tokens}")));
+        }
+    }
+
+    if let Some(max_batch_size) = max_batch_size {
+        if max_batch_size == 0 {
+            return Err(RouterError::ArgumentValidation(
+                "`max_batch_size` must be > 0".to_string(),
+            ));
+        }
+    }
+
+    let (backend, _backend_info) = connect_backend(
+        max_input_tokens,
+        max_total_tokens,
+        master_shard_uds_path,
+        waiting_served_ratio,
+        max_batch_prefill_tokens,
+        max_batch_total_tokens,
+        max_waiting_tokens,
+        max_batch_size,
+    )
+    .await?;
+
+    // Run server
+    server::run(
+        backend,
+        max_concurrent_requests,
+        max_best_of,
+        max_stop_sequences,
+        max_top_n_tokens,
+        max_input_tokens,
+        max_total_tokens,
+        validation_workers,
+        api_key,
+        tokenizer_name,
+        tokenizer_config_path,
+        revision,
+        hostname,
+        port,
+        cors_allow_origin,
+        ngrok,
+        ngrok_authtoken,
+        ngrok_edge,
+        messages_api_enabled,
+        disable_grammar_support,
+        max_client_batch_size,
+        usage_stats,
+    )
+    .await?;
+    Ok(())
+}
+
+#[derive(Debug, Error)]
+enum RouterError {
+    #[error("Argument validation error: {0}")]
+    ArgumentValidation(String),
+    #[error("Backend failed: {0}")]
+    Backend(#[from] V3Error),
+    #[error("WebServer error: {0}")]
+    WebServer(#[from] server::WebServerError),
+    #[error("Tokio runtime failed to start: {0}")]
+    Tokio(#[from] std::io::Error),
+}
diff --git a/backends/v3/src/queue.rs b/backends/v3/src/queue.rs
new file mode 100644
index 00000000..4ce54a79
--- /dev/null
+++ b/backends/v3/src/queue.rs
@@ -0,0 +1,824 @@
+use crate::block_allocator::{BlockAllocation, BlockAllocator};
+use crate::client;
+use crate::client::{
+    Batch, GrammarType, NextTokenChooserParameters, Request, StoppingCriteriaParameters,
+};
+use nohash_hasher::{BuildNoHashHasher, IntMap};
+use std::cmp::{max, min};
+use std::collections::VecDeque;
+use text_generation_router::infer::InferError;
+use text_generation_router::infer::InferStreamResponse;
+use text_generation_router::validation::{
+    Chunk, ChunksToString, ValidGenerateRequest, ValidGrammar, ValidParameters,
+    ValidStoppingParameters,
+};
+use tokio::sync::{mpsc, oneshot};
+use tokio::time::Instant;
+use tracing::{info_span, instrument, Instrument, Span};
+
+/// Queue entry
+#[derive(Debug)]
+pub(crate) struct Entry {
+    /// Request
+    pub request: ValidGenerateRequest,
+    /// Response sender to communicate between the Infer struct and the batching_task
+    pub response_tx: mpsc::UnboundedSender<Result<InferStreamResponse, InferError>>,
+    /// Span that will live as long as entry
+    pub span: Span,
+    /// Temporary span used as a guard when logging inference, wait times...
+    pub temp_span: Option<Span>,
+    /// Instant when this entry was queued
+    pub queue_time: Instant,
+    /// Instant when this entry was added to a batch
+    pub batch_time: Option<Instant>,
+    /// Block Allocation
+    pub block_allocation: Option<BlockAllocation>,
+}
+
+/// Request Queue
+#[derive(Debug, Clone)]
+pub(crate) struct Queue {
+    /// Channel to communicate with the background queue task
+    queue_sender: mpsc::UnboundedSender<QueueCommand>,
+}
+
+impl Queue {
+    pub(crate) fn new(
+        requires_padding: bool,
+        block_size: u32,
+        prefix_caching: bool,
+        window_size: Option<u32>,
+        speculate: u32,
+        max_input_tokens: u32,
+        max_total_tokens: u32,
+        max_batch_total_tokens: u32,
+    ) -> Self {
+        // Create channel
+        let (queue_sender, queue_receiver) = mpsc::unbounded_channel();
+
+        // Launch background queue task
+        tokio::spawn(queue_task(
+            requires_padding,
+            block_size,
+            prefix_caching,
+            window_size,
+            speculate,
+            max_input_tokens,
+            max_total_tokens,
+            max_batch_total_tokens,
+            queue_receiver,
+        ));
+
+        Self { queue_sender }
+    }
+
+    /// Append an entry to the queue
+    #[instrument(skip_all)]
+    pub(crate) fn append(&self, entry: Entry) {
+        // Send append command to the background task managing the state
+        // Unwrap is safe here
+        self.queue_sender
+            .send(QueueCommand::Append(Box::new(entry), Span::current()))
+            .unwrap();
+    }
+
+    // Get the next batch
+    #[instrument(skip(self))]
+    pub(crate) async fn next_batch(
+        &self,
+        min_size: Option<usize>,
+        max_size: Option<usize>,
+        prefill_token_budget: u32,
+        token_budget: u32,
+    ) -> Option<NextBatch> {
+        // Create response channel
+        let (response_sender, response_receiver) = oneshot::channel();
+        // Send next batch command to the background task managing the state
+        // Unwrap is safe here
+        self.queue_sender
+            .send(QueueCommand::NextBatch {
+                min_size,
+                max_size,
+                prefill_token_budget,
+                token_budget,
+                response_sender,
+                span: Span::current(),
+            })
+            .unwrap();
+        // Await on response channel
+        // Unwrap is safe here
+        response_receiver.await.unwrap()
+    }
+}
+
+// Background task responsible of the queue state
+async fn queue_task(
+    requires_padding: bool,
+    block_size: u32,
+    prefix_caching: bool,
+    window_size: Option<u32>,
+    speculate: u32,
+    max_input_tokens: u32,
+    max_total_tokens: u32,
+    max_batch_total_tokens: u32,
+    mut receiver: mpsc::UnboundedReceiver<QueueCommand>,
+) {
+    let mut state = State::new(
+        requires_padding,
+        block_size,
+        prefix_caching,
+        window_size,
+        speculate,
+        max_input_tokens,
+        max_total_tokens,
+        max_batch_total_tokens,
+    );
+
+    while let Some(cmd) = receiver.recv().await {
+        match cmd {
+            QueueCommand::Append(entry, span) => {
+                span.in_scope(|| state.append(*entry));
+                metrics::gauge!("tgi_queue_size").increment(1.0);
+            }
+            QueueCommand::NextBatch {
+                min_size,
+                max_size,
+                prefill_token_budget,
+                token_budget,
+                response_sender,
+                span,
+            } => {
+                let next_batch = state
+                    .next_batch(min_size, max_size, prefill_token_budget, token_budget)
+                    .instrument(span)
+                    .await;
+                response_sender.send(next_batch).unwrap();
+                metrics::gauge!("tgi_queue_size").set(state.entries.len() as f64);
+            }
+        }
+    }
+}
+
+/// Queue State
+#[derive(Debug)]
+struct State {
+    /// Queue entries organized in a Vec
+    entries: VecDeque<(u64, Entry)>,
+
+    /// Id of the next entry
+    next_id: u64,
+
+    /// Id of the next batch
+    next_batch_id: u64,
+
+    /// Paged Attention block size
+    block_size: u32,
+
+    /// Sliding window
+    window_size: Option<u32>,
+
+    /// Speculation amount
+    speculate: u32,
+
+    /// Paged Attention Block Allocation
+    block_allocator: Option<BlockAllocator>,
+
+    /// Require padding
+    requires_padding: bool,
+
+    /// max input tokens
+    max_input_tokens: u32,
+
+    /// max total tokens,
+    max_total_tokens: u32,
+}
+
+impl State {
+    fn new(
+        requires_padding: bool,
+        block_size: u32,
+        prefix_caching: bool,
+        window_size: Option<u32>,
+        speculate: u32,
+        max_input_tokens: u32,
+        max_total_tokens: u32,
+        max_batch_total_tokens: u32,
+    ) -> Self {
+        let block_allocator = (!requires_padding).then(|| {
+            BlockAllocator::new(
+                max_batch_total_tokens,
+                block_size,
+                prefix_caching,
+                window_size,
+            )
+        });
+
+        Self {
+            entries: VecDeque::with_capacity(128),
+            next_id: 0,
+            next_batch_id: 0,
+            block_size,
+            window_size,
+            speculate,
+            block_allocator,
+            requires_padding,
+            max_input_tokens,
+            max_total_tokens,
+        }
+    }
+
+    /// Append an entry to the queue
+    fn append(&mut self, mut entry: Entry) {
+        // Create a span that will live as long as the entry is in the queue waiting to be batched
+        let queue_span = info_span!(parent: &entry.span, "queued");
+        entry.temp_span = Some(queue_span);
+
+        // Push entry in the queue
+        self.entries.push_back((self.next_id, entry));
+        self.next_id += 1;
+    }
+
+    // Get the next batch
+    async fn next_batch(
+        &mut self,
+        min_size: Option<usize>,
+        max_size: Option<usize>,
+        prefill_token_budget: u32,
+        token_budget: u32,
+    ) -> Option<NextBatch> {
+        if self.entries.is_empty() {
+            tracing::debug!("No queue");
+            return None;
+        }
+
+        // Check if we have enough entries
+        if let Some(min_size) = min_size {
+            if self.entries.len() < min_size {
+                tracing::debug!("Not enough entries");
+                return None;
+            }
+        }
+
+        if let Some(max_size) = max_size {
+            if max_size == 0 {
+                tracing::debug!("No capacity");
+                return None;
+            }
+        }
+
+        // Pad prefill_token_budget to be a multiple of block size
+        let prefill_token_budget =
+            ((prefill_token_budget + self.block_size - 1) / self.block_size) * self.block_size;
+
+        // Create span for this batch to add context to inference calls
+        let next_batch_span = info_span!(parent: None, "batch", batch_size = tracing::field::Empty);
+        next_batch_span.follows_from(Span::current());
+
+        let mut batch = Vec::with_capacity(self.entries.len());
+        let mut max_input_length = 0;
+        let mut prefill_tokens: u32 = 0;
+        let mut decode_tokens: u32 = 0;
+        let mut max_blocks = 0;
+
+        // Pop entries starting from the front of the queue
+        'entry_loop: while let Some((id, entry)) = self.entries.pop_front() {
+            // Filter entries where the response receiver was dropped (== entries where the request
+            // was dropped by the client)
+            if entry.response_tx.is_closed() {
+                metrics::counter!("tgi_request_failure", "err" => "dropped").increment(1);
+                tracing::debug!("Dropping entry");
+                continue;
+            }
+
+            let block_allocation = match &self.block_allocator {
+                None => {
+                    // We pad to max input length in the Python shards
+                    // We need to take these padding tokens into the equation
+                    if self.requires_padding {
+                        prefill_tokens = (batch.len() + 1) as u32 * self.max_input_tokens;
+                    } else{
+                        max_input_length = max_input_length.max(entry.request.input_length);
+                        prefill_tokens = (batch.len() + 1) as u32 * max_input_length;
+                    }
+
+                    if self.requires_padding {
+                        decode_tokens = (batch.len() + 1) as u32 * (self.max_total_tokens - self.max_input_tokens);
+                    } else {
+                        decode_tokens += entry.request.stopping_parameters.max_new_tokens;
+                    }
+
+                    let total_tokens = prefill_tokens + decode_tokens + self.speculate;
+
+                    if prefill_tokens > prefill_token_budget || total_tokens > token_budget {
+                        // Entry is over budget
+                        // Add it back to the front
+                        tracing::debug!("Over budget: prefill_tokens={prefill_tokens} > {prefill_token_budget} || {prefill_tokens} + {decode_tokens} + {} > {token_budget}", self.speculate);
+                        self.entries.push_front((id, entry));
+                        break 'entry_loop;
+                    }
+                    None
+                }
+                Some(_block_allocator) => {
+                    prefill_tokens += entry.request.input_length;
+                    let max_new_tokens = match self.window_size {
+                        None => entry.request.stopping_parameters.max_new_tokens,
+                        Some(window_size) => min(
+                            window_size.saturating_sub(entry.request.input_length),
+                            entry.request.stopping_parameters.max_new_tokens,
+                        ),
+                    };
+                    decode_tokens += max_new_tokens;
+
+                    if prefill_tokens > prefill_token_budget
+                        || (prefill_tokens + decode_tokens + self.speculate) > token_budget
+                    {
+                        // Entry is over budget
+                        // Add it back to the front
+                        tracing::debug!("Over budget: prefill_tokens={prefill_tokens} > {prefill_token_budget} || {prefill_tokens} + {decode_tokens} + {} > {token_budget}", self.speculate);
+                        self.entries.push_front((id, entry));
+                        break;
+                    }
+
+                    let tokens = entry.request.input_length
+                        + entry.request.stopping_parameters.max_new_tokens
+                        + self.speculate
+                        - 1;
+
+                    // If users wants the prefill logprobs, we cannot reuse the cache.
+                    // So no input_ids for the radix tree.
+                    let input_ids = if entry.request.decoder_input_details {
+                        None
+                    } else {
+                        entry.request.input_ids.clone()
+                    };
+
+                    Some((tokens, input_ids))
+                }
+            };
+            batch.push((id, entry, block_allocation));
+            if Some(batch.len()) == max_size {
+                break;
+            }
+        }
+
+        // Empty batch
+        if batch.is_empty() {
+            tracing::debug!("Filterered out all entries");
+            return None;
+        }
+
+        // XXX We haven't allocated yet, so we're allowed to ditch the results.
+        // Check if our batch is big enough
+        if let Some(min_size) = min_size {
+            // Batch is too small
+            if batch.len() < min_size {
+                // Add back entries to the queue in the correct order
+                for (id, entry, _) in batch.into_iter().rev() {
+                    self.entries.push_front((id, entry));
+                }
+                return None;
+            }
+        }
+
+        let mut batch_requests = Vec::with_capacity(self.entries.len());
+        let mut batch_entries =
+            IntMap::with_capacity_and_hasher(self.entries.len(), BuildNoHashHasher::default());
+
+        for (id, mut entry, block_allocation) in batch {
+            let block_allocation = if let (Some((tokens, input_ids)), Some(block_allocator)) =
+                (block_allocation, &self.block_allocator)
+            {
+                tracing::debug!("Allocating {tokens} with {input_ids:?}");
+                match block_allocator.allocate(tokens, input_ids).await {
+                    None => {
+                        // Entry is over budget
+                        // Add it back to the front
+                        tracing::debug!("Over budget: not enough free blocks");
+                        self.entries.push_front((id, entry));
+                        continue;
+                    }
+                    Some(block_allocation) => {
+                        tracing::debug!("Allocation: {block_allocation:?}");
+                        max_blocks = max(max_blocks, block_allocation.blocks.len() as u32);
+                        Some(block_allocation)
+                    }
+                }
+            } else {
+                None
+            };
+            tracing::debug!("Accepting entry");
+            // Create a new span to link the batch back to this entry
+            let entry_batch_span = info_span!(parent: &entry.span, "infer");
+            // Add relationships
+            next_batch_span.follows_from(&entry_batch_span);
+            entry_batch_span.follows_from(&next_batch_span);
+            // Update entry
+            entry.temp_span = Some(entry_batch_span);
+
+            let (blocks, slots, prefix_len) = match &block_allocation {
+                None => (Vec::new(), Vec::new(), 0),
+                Some(block_allocation) => (
+                    block_allocation.blocks.clone(),
+                    block_allocation.slots.clone(),
+                    block_allocation.prefix_len,
+                ),
+            };
+
+            entry.block_allocation = block_allocation;
+
+            batch_requests.push(Request {
+                id,
+                prefill_logprobs: entry.request.decoder_input_details,
+                input_chunks: Some(client::Input {
+                    chunks: entry
+                        .request
+                        .inputs
+                        .clone()
+                        .into_iter()
+                        .map(|c| client::InputChunk {
+                            chunk: Some(match c {
+                                Chunk::Text(text) => client::Chunk::Text(text),
+                                Chunk::Image(image) => client::Chunk::Image(client::Image {
+                                    data: image.data,
+                                    mimetype: image.mimetype,
+                                }),
+                            }),
+                        })
+                        .collect(),
+                }),
+                inputs: entry.request.inputs.chunks_to_string(),
+                truncate: entry.request.truncate,
+                add_special_tokens: entry.request.add_special_tokens,
+                parameters: Some(NextTokenChooserParameters::from(
+                    entry.request.parameters.clone(),
+                )),
+                stopping_parameters: Some(StoppingCriteriaParameters::from(
+                    entry.request.stopping_parameters.clone(),
+                )),
+                top_n_tokens: entry.request.top_n_tokens,
+                blocks,
+                slots,
+                prefix_len,
+                adapter_id: entry.request.adapter_id.clone(),
+            });
+            // Set batch_time
+            entry.batch_time = Some(Instant::now());
+            // Insert in batch_entries IntMap
+            batch_entries.insert(id, entry);
+        }
+
+        // Empty batch
+        if batch_requests.is_empty() {
+            tracing::debug!("Filterered out all entries");
+            return None;
+        }
+
+        // Final batch size
+        let size = batch_requests.len() as u32;
+        next_batch_span.record("batch_size", size);
+
+        let batch = Batch {
+            id: self.next_batch_id,
+            requests: batch_requests,
+            size,
+            max_tokens: (prefill_tokens + decode_tokens),
+            max_blocks,
+        };
+        // Increment batch id
+        self.next_batch_id += 1;
+
+        metrics::histogram!("tgi_batch_next_size").record(batch.size as f64);
+
+        Some((batch_entries, batch, next_batch_span))
+    }
+}
+
+type NextBatch = (IntMap<u64, Entry>, Batch, Span);
+
+#[derive(Debug)]
+enum QueueCommand {
+    Append(Box<Entry>, Span),
+    NextBatch {
+        min_size: Option<usize>,
+        max_size: Option<usize>,
+        prefill_token_budget: u32,
+        token_budget: u32,
+        response_sender: oneshot::Sender<Option<NextBatch>>,
+        span: Span,
+    },
+}
+
+impl From<ValidParameters> for NextTokenChooserParameters {
+    fn from(value: ValidParameters) -> Self {
+        let (grammar, grammar_type) = match value.grammar {
+            None => (String::new(), GrammarType::None),
+
+            Some(grammar) => match grammar {
+                ValidGrammar::Json(grammar_string) => (grammar_string, GrammarType::Json),
+                ValidGrammar::Regex(grammar_string) => (grammar_string, GrammarType::Regex),
+            },
+        };
+
+        Self {
+            temperature: value.temperature,
+            top_k: value.top_k,
+            top_p: value.top_p,
+            typical_p: value.typical_p,
+            do_sample: value.do_sample,
+            seed: value.seed,
+            repetition_penalty: value.repetition_penalty,
+            frequency_penalty: value.frequency_penalty,
+            watermark: value.watermark,
+            grammar,
+            grammar_type: grammar_type.into(),
+        }
+    }
+}
+
+impl From<ValidStoppingParameters> for StoppingCriteriaParameters {
+    fn from(value: ValidStoppingParameters) -> Self {
+        Self {
+            max_new_tokens: value.max_new_tokens,
+            stop_sequences: value.stop_sequences,
+            ignore_eos_token: value.ignore_eos_token,
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use super::*;
+    use tracing::info_span;
+
+    fn default_entry() -> (
+        Entry,
+        mpsc::UnboundedReceiver<Result<InferStreamResponse, InferError>>,
+    ) {
+        let (response_tx, receiver_tx) = mpsc::unbounded_channel();
+
+        let entry = Entry {
+            request: ValidGenerateRequest {
+                inputs: vec![],
+                input_ids: Some(Arc::new(vec![])),
+                input_length: 0,
+                add_special_tokens: true,
+                truncate: 0,
+                decoder_input_details: false,
+                parameters: ValidParameters {
+                    temperature: 0.0,
+                    top_k: 0,
+                    top_p: 0.0,
+                    typical_p: 0.0,
+                    do_sample: false,
+                    seed: 0,
+                    repetition_penalty: 0.0,
+                    frequency_penalty: 0.0,
+                    watermark: false,
+                    grammar: None,
+                },
+                stopping_parameters: ValidStoppingParameters {
+                    ignore_eos_token: false,
+                    max_new_tokens: 1,
+                    stop_sequences: vec![],
+                },
+                top_n_tokens: 0,
+                adapter_id: None,
+            },
+            response_tx,
+            span: info_span!("entry"),
+            temp_span: None,
+            queue_time: Instant::now(),
+            batch_time: None,
+            block_allocation: None,
+        };
+        (entry, receiver_tx)
+    }
+
+    #[tokio::test]
+    async fn test_append() {
+        let mut state = State::new(false, 1, false, None, 0, 16);
+        let (entry, _guard) = default_entry();
+
+        assert_eq!(state.next_id, 0);
+        assert_eq!(state.entries.len(), 0);
+
+        state.append(entry);
+
+        assert_eq!(state.next_id, 1);
+        assert_eq!(state.entries.len(), 1);
+        let (id, _) = state.entries.remove(0).unwrap();
+        assert_eq!(id, 0);
+    }
+
+    #[tokio::test]
+    async fn test_next_batch_empty() {
+        let mut state = State::new(false, 1, false, None, 0, 16);
+
+        assert!(state.next_batch(None, None, 1, 1).await.is_none());
+        assert!(state.next_batch(Some(1), None, 1, 1).await.is_none());
+    }
+
+    #[tokio::test]
+    async fn test_next_batch_min_size() {
+        let mut state = State::new(false, 1, false, None, 0, 16);
+        let (entry1, _guard1) = default_entry();
+        let (entry2, _guard2) = default_entry();
+        state.append(entry1);
+        state.append(entry2);
+
+        let (entries, batch, _) = state.next_batch(None, None, 2, 2).await.unwrap();
+        assert_eq!(entries.len(), 2);
+        assert!(entries.contains_key(&0));
+        assert!(entries.contains_key(&1));
+        assert!(entries.get(&0).unwrap().batch_time.is_some());
+        assert!(entries.get(&1).unwrap().batch_time.is_some());
+        assert_eq!(batch.id, 0);
+        assert_eq!(batch.size, 2);
+
+        assert_eq!(state.next_id, 2);
+        assert_eq!(state.entries.len(), 0);
+        assert_eq!(state.next_batch_id, 1);
+
+        let (entry3, _guard3) = default_entry();
+        state.append(entry3);
+
+        assert!(state.next_batch(Some(2), None, 2, 2).await.is_none());
+
+        assert_eq!(state.next_id, 3);
+        assert_eq!(state.entries.len(), 1);
+        let (id, _) = state.entries.remove(0).unwrap();
+        assert_eq!(id, 2);
+    }
+
+    #[tokio::test]
+    async fn test_next_batch_max_size() {
+        let mut state = State::new(false, 1, false, None, 0, 16);
+        let (entry1, _guard1) = default_entry();
+        let (entry2, _guard2) = default_entry();
+        state.append(entry1);
+        state.append(entry2);
+
+        let (entries, batch, _) = state.next_batch(None, Some(1), 2, 2).await.unwrap();
+        assert_eq!(entries.len(), 1);
+        assert!(entries.contains_key(&0));
+        assert!(entries.get(&0).unwrap().batch_time.is_some());
+        assert_eq!(batch.id, 0);
+        assert_eq!(batch.size, 1);
+
+        assert_eq!(state.next_id, 2);
+        assert_eq!(state.entries.len(), 1);
+        assert_eq!(state.next_batch_id, 1);
+    }
+
+    #[tokio::test]
+    async fn test_next_batch_token_budget() {
+        let mut state = State::new(false, 1, false, None, 0, 2);
+        let (entry1, _guard1) = default_entry();
+        let (entry2, _guard2) = default_entry();
+        state.append(entry1);
+        state.append(entry2);
+
+        let (entries, batch, _) = state.next_batch(None, None, 1, 1).await.unwrap();
+        assert_eq!(entries.len(), 1);
+        assert!(entries.contains_key(&0));
+        assert_eq!(batch.id, 0);
+        assert_eq!(batch.size, 1);
+
+        assert_eq!(state.next_id, 2);
+        assert_eq!(state.entries.len(), 1);
+        assert_eq!(state.next_batch_id, 1);
+
+        let (entry3, _guard3) = default_entry();
+        state.append(entry3);
+
+        let (entries, batch, _) = state.next_batch(None, None, 3, 3).await.unwrap();
+        assert_eq!(entries.len(), 2);
+        assert!(entries.contains_key(&1));
+        assert!(entries.contains_key(&2));
+        assert_eq!(batch.id, 1);
+        assert_eq!(batch.size, 2);
+
+        assert_eq!(state.next_id, 3);
+        assert_eq!(state.entries.len(), 0);
+        assert_eq!(state.next_batch_id, 2);
+    }
+
+    #[tokio::test]
+    async fn test_queue_append() {
+        let queue = Queue::new(false, 1, false, None, 0, 16);
+        let (entry, _guard) = default_entry();
+        queue.append(entry);
+    }
+
+    #[tokio::test]
+    async fn test_queue_next_batch_empty() {
+        let queue = Queue::new(false, 1, false, None, 0, 16);
+
+        assert!(queue.next_batch(None, None, 1, 1).await.is_none());
+        assert!(queue.next_batch(Some(1), None, 1, 1).await.is_none());
+    }
+
+    #[tokio::test]
+    async fn test_queue_next_batch_min_size() {
+        let queue = Queue::new(false, 1, false, None, 0, 16);
+        let (entry1, _guard1) = default_entry();
+        let (entry2, _guard2) = default_entry();
+        queue.append(entry1);
+        queue.append(entry2);
+
+        let (entries, batch, _) = queue.next_batch(None, None, 2, 2).await.unwrap();
+        assert_eq!(entries.len(), 2);
+        assert!(entries.contains_key(&0));
+        assert!(entries.contains_key(&1));
+        assert!(entries.get(&0).unwrap().batch_time.is_some());
+        assert!(entries.get(&1).unwrap().batch_time.is_some());
+        assert_eq!(batch.id, 0);
+        assert_eq!(batch.size, 2);
+
+        let (entry3, _guard3) = default_entry();
+        queue.append(entry3);
+
+        // Not enough requests pending
+        assert!(queue.next_batch(Some(2), None, 2, 2).await.is_none());
+        // Not enough token budget
+        assert!(queue.next_batch(Some(1), None, 0, 0).await.is_none());
+        // Ok
+        let (entries2, batch2, _) = queue.next_batch(Some(1), None, 2, 2).await.unwrap();
+        assert_eq!(entries2.len(), 1);
+        assert!(entries2.contains_key(&2));
+        assert!(entries2.get(&2).unwrap().batch_time.is_some());
+        assert_eq!(batch2.id, 1);
+        assert_eq!(batch2.size, 1);
+    }
+
+    #[tokio::test]
+    async fn test_queue_next_batch_max_size() {
+        let queue = Queue::new(false, 1, false, None, 0, 16);
+        let (entry1, _guard1) = default_entry();
+        let (entry2, _guard2) = default_entry();
+        queue.append(entry1);
+        queue.append(entry2);
+
+        let (entries, batch, _) = queue.next_batch(None, Some(1), 2, 2).await.unwrap();
+        assert_eq!(entries.len(), 1);
+        assert!(entries.contains_key(&0));
+        assert!(entries.get(&0).unwrap().batch_time.is_some());
+        assert_eq!(batch.id, 0);
+        assert_eq!(batch.size, 1);
+    }
+
+    #[tokio::test]
+    async fn test_queue_next_batch_token_budget() {
+        let queue = Queue::new(false, 1, false, None, 0, 16);
+        let (entry1, _guard1) = default_entry();
+        let (entry2, _guard2) = default_entry();
+        queue.append(entry1);
+        queue.append(entry2);
+
+        let (entries, batch, _) = queue.next_batch(None, None, 1, 1).await.unwrap();
+        assert_eq!(entries.len(), 1);
+        assert!(entries.contains_key(&0));
+        assert_eq!(batch.id, 0);
+        assert_eq!(batch.size, 1);
+
+        let (entry3, _guard3) = default_entry();
+        queue.append(entry3);
+
+        let (entries, batch, _) = queue.next_batch(None, None, 3, 3).await.unwrap();
+        assert_eq!(entries.len(), 2);
+        assert!(entries.contains_key(&1));
+        assert!(entries.contains_key(&2));
+        assert_eq!(batch.id, 1);
+        assert_eq!(batch.size, 2);
+    }
+
+    #[tokio::test]
+    async fn test_queue_next_batch_token_speculate() {
+        let queue = Queue::new(false, 1, false, None, 2, 16);
+        let (entry1, _guard1) = default_entry();
+        let (entry2, _guard2) = default_entry();
+        queue.append(entry1);
+        queue.append(entry2);
+
+        // Budget of 1 is not enough
+        assert!(queue.next_batch(None, None, 1, 1).await.is_none());
+
+        let (entries, batch, _) = queue.next_batch(None, None, 6, 6).await.unwrap();
+        assert_eq!(entries.len(), 2);
+        assert!(entries.contains_key(&0));
+        assert!(entries.contains_key(&1));
+        assert_eq!(batch.id, 0);
+        assert_eq!(batch.size, 2);
+    }
+
+    #[tokio::test]
+    async fn test_queue_next_batch_dropped_receiver() {
+        let queue = Queue::new(false, 1, false, None, 0, 16);
+        let (entry, _) = default_entry();
+        queue.append(entry);
+
+        assert!(queue.next_batch(None, None, 1, 1).await.is_none());
+    }
+}
diff --git a/backends/v3/src/radix.rs b/backends/v3/src/radix.rs
new file mode 100644
index 00000000..8a544891
--- /dev/null
+++ b/backends/v3/src/radix.rs
@@ -0,0 +1,876 @@
+use crate::block_allocator::{Allocator, BlockAllocation};
+use slotmap::{DefaultKey, SlotMap};
+use std::hash::{Hash, Hasher};
+use std::{
+    collections::{BTreeSet, HashMap},
+    sync::Arc,
+};
+
+fn hash(slice: &[u32]) -> u64 {
+    assert!(!slice.is_empty());
+    if slice.len() == 1 {
+        slice[0] as u64
+    } else {
+        let mut s = std::hash::DefaultHasher::new();
+        slice.hash(&mut s);
+        s.finish()
+    }
+}
+
+pub struct RadixAllocator {
+    allocation_id: u64,
+
+    allocations: HashMap<u64, RadixAllocation>,
+
+    cache_blocks: RadixTrie,
+
+    /// Blocks that are immediately available for allocation.
+    free_blocks: Vec<u32>,
+
+    #[allow(dead_code)]
+    // This isn't used because the prefix need to match without the windowing
+    // mecanism. This at worst is overallocating, not necessarily being wrong.
+    window_size: Option<u32>,
+
+    block_size: u32,
+}
+
+impl RadixAllocator {
+    pub fn new(block_size: u32, n_blocks: u32, window_size: Option<u32>) -> Self {
+        RadixAllocator {
+            allocation_id: 0,
+            allocations: HashMap::new(),
+            cache_blocks: RadixTrie::new(block_size as usize),
+
+            // Block 0 is reserved for health checks.
+            free_blocks: (1..n_blocks).collect(),
+            window_size,
+            block_size,
+        }
+    }
+
+    fn alloc_or_reclaim(&mut self, n_blocks_needed: usize) -> Option<Vec<u32>> {
+        if self.free_blocks.len() < n_blocks_needed {
+            // This is a bit annoying, we first extend the free list and then
+            // split it off again below. This is because we need to put it on
+            // the free list if we cannot allocate enough blocks. This is only
+            // temporary, the trie needs to be able to report whether it can
+            // allocate the requested amount. Just not implemented yet.
+            tracing::debug!(
+                "Free blocks {}  need {n_blocks_needed}",
+                self.free_blocks.len()
+            );
+            self.free_blocks.extend(
+                self.cache_blocks
+                    .evict(n_blocks_needed - self.free_blocks.len()),
+            );
+        }
+
+        if self.free_blocks.len() >= n_blocks_needed {
+            Some(
+                self.free_blocks
+                    .split_off(self.free_blocks.len() - n_blocks_needed),
+            )
+        } else {
+            None
+        }
+    }
+}
+
+// Allocator trait
+impl Allocator for RadixAllocator {
+    fn allocate(
+        &mut self,
+        tokens: u32,
+        prefill_tokens: Option<Arc<Vec<u32>>>,
+    ) -> Option<BlockAllocation> {
+        let mut blocks = vec![];
+        let prefix_node = if let Some(prefill_tokens) = prefill_tokens.as_ref() {
+            let node_id = self
+                .cache_blocks
+                .find(prefill_tokens.as_slice(), &mut blocks);
+            node_id
+        } else {
+            self.cache_blocks.root_id()
+        };
+
+        // Even if this allocation fails below, we need to increase he
+        // refcount to ensure that the prefix that was found is not evicted.
+        self.cache_blocks
+            .incref(prefix_node)
+            .expect("Failed to increment refcount");
+
+        let prefix_len = blocks.len() * self.block_size as usize;
+        let suffix_len = tokens - prefix_len as u32;
+
+        let suffix_blocks = (suffix_len + self.block_size - 1) / self.block_size;
+
+        tracing::info!("Prefix {prefix_len} - Suffix {suffix_len}");
+
+        match self.alloc_or_reclaim(suffix_blocks as usize) {
+            Some(suffix_blocks) => blocks.extend(suffix_blocks),
+            None => {
+                tracing::debug!("Cannot allocate {:?}", self.cache_blocks);
+                tracing::debug!("Found {prefix_len} prefix tokens need {suffix_blocks} suffix blocks for {tokens} tokens");
+                tracing::debug!("Block size {}", self.block_size);
+                self.cache_blocks
+                    .decref(prefix_node)
+                    .expect("Failed to decrement refcount");
+                return None;
+            }
+        }
+
+        // 1:1 mapping of blocks and slots.
+        let slots = if self.block_size == 1 {
+            blocks.clone()
+        } else {
+            let mut slots = Vec::with_capacity(blocks.len() * self.block_size as usize);
+            'slots: for block_id in &blocks {
+                for s in (block_id * self.block_size)..((block_id + 1) * self.block_size) {
+                    slots.push(s);
+                    if slots.len() as u32 == tokens {
+                        break 'slots;
+                    }
+                }
+            }
+            slots
+        };
+
+        let allocation = RadixAllocation {
+            prefix_node,
+            cached_prefix_len: prefix_len,
+            prefill_tokens: prefill_tokens.clone(),
+        };
+
+        self.allocation_id += 1;
+        self.allocations.insert(self.allocation_id, allocation);
+
+        Some(BlockAllocation {
+            allocation_id: self.allocation_id,
+            block_allocator: None,
+            blocks,
+            slots,
+            prefix_len: prefix_len as u32,
+        })
+    }
+
+    fn free(&mut self, blocks: Vec<u32>, allocation_id: u64) {
+        let allocation = match self.allocations.remove(&allocation_id) {
+            Some(allocation) => allocation,
+            None => unreachable!("Tried to free an unknown allocation."),
+        };
+
+        self.cache_blocks
+            .decref(allocation.prefix_node)
+            .expect("Failed to decrement refcount");
+
+        if let Some(prefill_tokens) = allocation.prefill_tokens {
+            let prefill_tokens = prefill_tokens.as_slice();
+
+            // If there are prefill tokens that did not come from the cache,
+            // add them to the cache.
+            if prefill_tokens.len() > allocation.cached_prefix_len {
+                let aligned =
+                    (prefill_tokens.len() / self.block_size as usize) * self.block_size as usize;
+                if aligned > 0 {
+                    let prefix_len = self
+                        .cache_blocks
+                        .insert(
+                            &prefill_tokens[..aligned],
+                            &blocks[..aligned / self.block_size as usize],
+                        )
+                        // Unwrap, failing is a programming error.
+                        .expect("Failed to store prefill tokens");
+                    // We can have a prefill with the following structure:
+                    //
+                    // |---| From the prefix cache.
+                    // A B C D E F G
+                    //|--------| Found in the trie during insertion.
+                    //
+                    // This means that while processing this request there was a
+                    // partially overlapping request that had A..=E in its
+                    // prefill. In this case we need to free the blocks D E.
+                    if prefix_len > allocation.cached_prefix_len {
+                        self.free_blocks.extend(
+                            &blocks[allocation.cached_prefix_len / self.block_size as usize
+                                ..prefix_len / self.block_size as usize],
+                        );
+                    }
+                }
+            }
+
+            // Free non-prefill blocks.
+            self.free_blocks
+                .extend(&blocks[prefill_tokens.len() / self.block_size as usize..]);
+        } else {
+            self.free_blocks.extend(blocks);
+        }
+    }
+}
+
+struct RadixAllocation {
+    prefix_node: NodeId,
+    cached_prefix_len: usize,
+    prefill_tokens: Option<Arc<Vec<u32>>>,
+}
+
+// Radix trie that is heavily inspired by radix attention from sglang.
+//
+// The trie is optimized for prefix caching:
+//
+// - A normal radix trie stores discrete values. In this radix trie,
+//   inserting *abc* with value *xyz* will also enable lookup for
+//   *a* (*x*) and *ab* (*xy*).
+// - As a result, every value is required to have the same length as
+//   the key.
+// - We store additional information in each node, such as last access
+//   time and a reference count.
+
+#[derive(Debug)]
+pub enum TrieError {
+    InvalidNodeId,
+    RefCountUnderflow,
+}
+
+pub type NodeId = DefaultKey;
+
+#[derive(Debug)]
+pub struct RadixTrie {
+    /// Identifier of the root nod.
+    root: DefaultKey,
+
+    /// Leave node identifiers ordered by increasing recency.
+    leaves: BTreeSet<(u64, NodeId)>,
+
+    /// All trie nodes.
+    nodes: SlotMap<NodeId, TrieNode>,
+
+    /// Time as a monotonically increating counter to avoid the system
+    /// call that a real time lookup would require.
+    time: u64,
+
+    /// All blocks need to be aligned with this
+    block_size: usize,
+}
+
+impl RadixTrie {
+    /// Construct a new radix trie.
+    pub fn new(block_size: usize) -> Self {
+        let root = TrieNode::new(vec![], vec![], 0, None);
+        let mut nodes = SlotMap::new();
+        let root = nodes.insert(root);
+        RadixTrie {
+            leaves: BTreeSet::new(),
+            nodes,
+            root,
+            time: 0,
+            block_size,
+        }
+    }
+
+    /// Find the prefix of the given tokens.
+    ///
+    /// The blocks corresponding to the part of the prefix that could be found
+    /// are written to `blocks`. The number of blocks is in `0..=tokens.len()`.
+    /// Returns the identifier of the trie node that contains the longest
+    /// prefix. The node identifier can be used by callers to e.g. increase its
+    /// reference count.
+    ///
+    /// Using this method will update the access time of the traversed nodes.
+    pub fn find(&mut self, key: &[u32], blocks: &mut Vec<u32>) -> NodeId {
+        self.time += 1;
+        self.find_(self.root, key, blocks)
+    }
+
+    /// Find worker.
+    fn find_(&mut self, mut node_id: NodeId, key: &[u32], blocks: &mut Vec<u32>) -> NodeId {
+        let node = &self.nodes[node_id];
+
+        if key.len() >= self.block_size {
+            let node_key = hash(&key[..self.block_size]);
+            if let Some(&child_id) = node.children.get(&node_key) {
+                self.update_access_time(child_id);
+                let child = self.nodes.get(child_id).expect("Invalid child identifier");
+                let shared_prefix_len = shared_prefix(&child.key, key, self.block_size);
+                assert_eq!(shared_prefix_len % self.block_size, 0);
+                blocks.extend(&child.blocks[..shared_prefix_len / self.block_size]);
+
+                let key = &key[shared_prefix_len..];
+                if !key.is_empty() {
+                    node_id = self.find_(child_id, key, blocks);
+                }
+            }
+        }
+
+        node_id
+    }
+
+    /// Decrease the reference count of a node.
+    pub fn decref(&mut self, node_id: NodeId) -> Result<(), TrieError> {
+        // We don't care about refcounting for root, since it will never
+        // be evicted.
+        if node_id == self.root {
+            return Ok(());
+        }
+
+        let node = self
+            .nodes
+            .get_mut(node_id)
+            .ok_or(TrieError::InvalidNodeId)?;
+        if node.ref_count == 0 {
+            return Err(TrieError::RefCountUnderflow);
+        }
+
+        node.ref_count -= 1;
+        if node.ref_count == 0 {
+            assert!(
+                node.children.is_empty(),
+                "Nodes with children must have refcount > 0"
+            );
+
+            self.leaves.insert((node.last_accessed, node_id));
+        }
+
+        Ok(())
+    }
+
+    /// Increase the reference count of a node.
+    pub fn incref(&mut self, node_id: NodeId) -> Result<(), TrieError> {
+        if node_id == self.root {
+            return Ok(());
+        }
+
+        let node = self
+            .nodes
+            .get_mut(node_id)
+            .ok_or(TrieError::InvalidNodeId)?;
+        if node.ref_count == 0 {
+            self.leaves.remove(&(node.last_accessed, node_id));
+        }
+        node.ref_count += 1;
+
+        Ok(())
+    }
+
+    /// Evict `n_blocks` from the trie.
+    ///
+    /// Returns the evicted blocks. When the length is less than `n_blocks`,
+    /// not enough blocks could be evicted.
+    pub fn evict(&mut self, n_blocks: usize) -> Vec<u32> {
+        // NOTE: we don't return Result here. If any of the unwrapping fails,
+        // it's a programming error in the trie implementation, not a user
+        // error caused by e.g. an invalid argument.
+
+        // TODO: add some bookkeeping in the future to check whether we can
+        // evict n_blocks and return `None` if we can't. We are now needlessly
+        // evicting prefixes from the cache in such a case.
+        let mut evicted = Vec::new();
+        tracing::debug!("Evicting in search of {n_blocks}");
+
+        while let Some((last_access, node_id)) = self.leaves.pop_first() {
+            let blocks_needed = n_blocks.saturating_sub(evicted.len());
+            tracing::debug!("Evicting node {node_id:?} ");
+
+            let node = self.nodes.get(node_id).expect("Leave does not exist");
+            assert_eq!(
+                node.ref_count, 0,
+                "Leaf must have refcount of 0, got {}",
+                node.ref_count
+            );
+
+            if blocks_needed >= node.blocks.len() {
+                // We need to evict the whole node if we need more blocks than it has.
+                let node = self.remove_node(node_id);
+                evicted.extend(node.blocks);
+
+                if evicted.len() >= n_blocks {
+                    break;
+                }
+            } else {
+                // The node has more blocks than needed, so we'll just remove
+                // the required number of blocks and leave the remaining blocks
+                // untouched.
+                let node = self.nodes.get_mut(node_id).expect("Leave does not exist");
+
+                let truncate_blocks = node.blocks.len() - blocks_needed;
+                let truncate_tokens = truncate_blocks * self.block_size;
+                node.key.truncate(truncate_tokens);
+                evicted.extend(node.blocks.split_off(truncate_blocks));
+                self.leaves.insert((last_access, node_id));
+                break;
+            }
+        }
+
+        evicted
+    }
+
+    /// Insert a prefill along with its blocks.
+    ///
+    /// This method returns the length of the prefix that was already
+    /// in the trie. E.g. if the length is 10, this means that for
+    /// the first 10 elements of the tree **the blocks are not updated**.
+    pub fn insert(&mut self, tokens: &[u32], blocks: &[u32]) -> Result<usize, TrieError> {
+        self.time += 1;
+        let common = self.insert_(self.root, tokens, blocks)?;
+        Ok(common)
+    }
+
+    /// Insertion worker.
+    fn insert_(
+        &mut self,
+        node_id: NodeId,
+        tokens: &[u32],
+        blocks: &[u32],
+    ) -> Result<usize, TrieError> {
+        // TODO: in the future we may want to check that the blocks match for
+        // the part of the prefix that is already in the trie to detect
+        // mismatches.
+
+        assert_eq!(tokens.len(), blocks.len() * self.block_size);
+
+        let node_key = hash(&tokens[..self.block_size]);
+        if let Some(&child_id) = self.nodes[node_id].children.get(&node_key) {
+            self.update_access_time(child_id);
+            let child = self
+                .nodes
+                .get_mut(child_id)
+                // Unwrap here, since failure is a bug.
+                .expect("Child node does not exist");
+            let shared_prefix_len = shared_prefix(&child.key, tokens, self.block_size);
+
+            // We are done, the prefix is already in the trie.
+            if shared_prefix_len == tokens.len() || shared_prefix_len == 0 {
+                return Ok(shared_prefix_len);
+            }
+
+            // The node's prefix is a prefix of the insertion prefix.
+            if shared_prefix_len == child.key.len() {
+                return Ok(shared_prefix_len
+                    + self.insert_(
+                        child_id,
+                        &tokens[shared_prefix_len..],
+                        &blocks[shared_prefix_len / self.block_size..],
+                    )?);
+            }
+
+            // The node's prefix and the insertion prefix only match partially,
+            // split the node to just contain the matching part. Then insert the
+            // remainder of the prefix into the node again
+            let child_id = self.split_node(child_id, shared_prefix_len);
+            let key = &tokens[shared_prefix_len..];
+            let blocks = &blocks[shared_prefix_len / self.block_size..];
+            Ok(shared_prefix_len + self.insert_(child_id, key, blocks)?)
+        } else {
+            self.add_node(node_id, tokens, blocks);
+            Ok(0)
+        }
+    }
+
+    fn split_node(&mut self, node_id: NodeId, prefix_len: usize) -> NodeId {
+        // We have to make the current node a child to ensure that its
+        // properties and node id stay the same.
+
+        // This funcion unwraps, an  invalid node_id is a programming error.
+
+        let node = self
+            .nodes
+            .get_mut(node_id)
+            .expect("Node to-be split does not exist");
+        let mut parent_key = node.key.split_off(prefix_len);
+        let prefix_blocks = prefix_len / self.block_size;
+        let mut parent_blocks = node.blocks.split_off(prefix_blocks);
+
+        // Move first part of the prefix to the parent. We swap to avoid
+        // an allocation + copy for both splits of the key/blocks.
+        std::mem::swap(&mut node.key, &mut parent_key);
+        std::mem::swap(&mut node.blocks, &mut parent_blocks);
+
+        let node_key = hash(&node.key[..self.block_size]);
+
+        let grandparent_id = node.parent.expect("Node does not have a parent");
+        let parent_id = self.add_node(grandparent_id, parent_key, parent_blocks);
+        self.add_node_to_parent(parent_id, node_key, node_id);
+
+        // Reborrow to make the borrow checker happy.
+        let node = self
+            .nodes
+            .get_mut(node_id)
+            .expect("Node to-be split does not exist");
+        node.parent = Some(parent_id);
+
+        parent_id
+    }
+
+    /// Create a node and add it to the parent.
+    fn add_node(
+        &mut self,
+        parent_id: NodeId,
+        key: impl Into<Vec<u32>>,
+        blocks: impl Into<Vec<u32>>,
+    ) -> NodeId {
+        let key = key.into();
+        let blocks = blocks.into();
+        let first = hash(&key[..self.block_size]);
+
+        let child = TrieNode::new(key, blocks, self.time, Some(parent_id));
+        let child_id = self.nodes.insert(child);
+
+        self.add_node_to_parent(parent_id, first, child_id);
+        self.leaves.insert((self.time, child_id));
+
+        child_id
+    }
+
+    /// Add a node to the parent.
+    fn add_node_to_parent(&mut self, parent_id: NodeId, hash: u64, child_id: NodeId) {
+        // Unwrap here, passing in an unknown id is a programming error.
+        let parent = self.nodes.get_mut(parent_id).expect("Unknown parent node");
+        if parent.children.insert(hash, child_id).is_none() {
+            // Only increase reference count if child does not replace another child.
+            self.incref(parent_id)
+                .expect("Failed to increase parent refcount");
+        }
+    }
+
+    /// Remove a node from the trie.
+    fn remove_node(&mut self, node_id: NodeId) -> TrieNode {
+        // Unwrap here, passing in an unknown id is a programming error.
+        let node = self.nodes.remove(node_id).expect("Unknown node");
+        assert!(
+            node.children.is_empty(),
+            "Tried to remove a node with {} children",
+            node.children.len()
+        );
+        let parent_id = node.parent.expect("Attempted to remove root node");
+        let parent = self.nodes.get_mut(parent_id).expect("Unknown parent node");
+
+        let node_key = hash(&node.key[..self.block_size]);
+        parent.children.remove(&node_key);
+        self.decref(parent_id)
+            .expect("Failed to decrease parent refcount");
+        node
+    }
+
+    fn update_access_time(&mut self, node_id: NodeId) {
+        // Unwrap here, passing in an unknown id is a programming error.
+        let node = self.nodes.get_mut(node_id).expect("Unknown node");
+
+        // Update the ordered leaves set if the node is a leave.
+        if self.leaves.remove(&(node.last_accessed, node_id)) {
+            self.leaves.insert((self.time, node_id));
+        }
+
+        node.last_accessed = self.time;
+    }
+
+    #[allow(dead_code)]
+    #[doc(hidden)]
+    /// Print debugging output for the trie.
+    ///
+    /// In contrast to `Debug` nicely formatted.
+    pub fn print_debug(&self) {
+        self.print_debug_(self.root, 0);
+    }
+
+    fn print_debug_(&self, node_id: NodeId, indent: usize) {
+        let node = &self.nodes[node_id];
+        eprintln!(
+            "{}{:?}, key: {:?}, blocks: {:?}, ref_count: {}, last_accessed: {}, parent: {:?}, children: {:?}",
+            " ".repeat(indent),
+            node_id,
+            node.key,
+            node.blocks,
+            node.ref_count,
+            node.last_accessed,
+            node.parent,
+            node.children
+        );
+        for child_id in self.nodes[node_id].children.values() {
+            self.print_debug_(*child_id, indent + 2);
+        }
+    }
+
+    pub(crate) fn root_id(&self) -> DefaultKey {
+        self.root
+    }
+}
+
+/// Trie node.
+#[derive(Debug)]
+struct TrieNode {
+    blocks: Vec<u32>,
+    children: HashMap<u64, NodeId>,
+    key: Vec<u32>,
+    last_accessed: u64,
+    parent: Option<NodeId>,
+    ref_count: usize,
+}
+
+impl TrieNode {
+    fn new(key: Vec<u32>, blocks: Vec<u32>, last_accessed: u64, parent: Option<NodeId>) -> Self {
+        TrieNode {
+            children: HashMap::new(),
+            key,
+            blocks,
+            last_accessed,
+            parent,
+            ref_count: 0,
+        }
+    }
+}
+
+fn shared_prefix(left: &[u32], right: &[u32], block_size: usize) -> usize {
+    let full = left.iter().zip(right).take_while(|(a, b)| a == b).count();
+    // NOTE: this is the case because the child node was chosen based on
+    //       matching the first character of the key/prefix.
+    assert!(full > 0, "Prefixes must at least share 1 token");
+    (full / block_size) * block_size
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use super::*;
+
+    #[test]
+    fn allocator_block_size() {
+        let mut cache = RadixAllocator::new(2, 12, None);
+        let allocation = cache.allocate(8, Some(Arc::new(vec![0, 1, 2, 3]))).unwrap();
+        assert_eq!(allocation.blocks, vec![8, 9, 10, 11]);
+        assert_eq!(allocation.slots, vec![16, 17, 18, 19, 20, 21, 22, 23]);
+        assert_eq!(allocation.prefix_len, 0);
+        cache.free(allocation.blocks.clone(), allocation.allocation_id);
+
+        let allocation = cache.allocate(8, Some(Arc::new(vec![0, 1, 2, 3]))).unwrap();
+        assert_eq!(allocation.blocks, vec![8, 9, 10, 11]);
+        assert_eq!(allocation.slots, vec![16, 17, 18, 19, 20, 21, 22, 23]);
+        assert_eq!(allocation.prefix_len, 4);
+    }
+
+    #[test]
+    fn allocator_block_size_non_aligned() {
+        let mut cache = RadixAllocator::new(2, 12, None);
+        let allocation = cache.allocate(7, Some(Arc::new(vec![0, 1, 2]))).unwrap();
+        assert_eq!(allocation.blocks, vec![8, 9, 10, 11]);
+        assert_eq!(allocation.slots, vec![16, 17, 18, 19, 20, 21, 22]);
+        assert_eq!(allocation.prefix_len, 0);
+        cache.free(allocation.blocks.clone(), allocation.allocation_id);
+
+        let allocation = cache.allocate(7, Some(Arc::new(vec![0, 1, 2]))).unwrap();
+        assert_eq!(allocation.blocks, vec![8, 9, 10, 11]);
+        assert_eq!(allocation.slots, vec![16, 17, 18, 19, 20, 21, 22]);
+        assert_eq!(allocation.prefix_len, 2);
+    }
+
+    #[test]
+    fn allocator_reuses_prefixes() {
+        let mut cache = RadixAllocator::new(1, 12, None);
+        let allocation = cache.allocate(8, Some(Arc::new(vec![0, 1, 2, 3]))).unwrap();
+        assert_eq!(allocation.blocks, vec![4, 5, 6, 7, 8, 9, 10, 11]);
+        assert_eq!(allocation.blocks, allocation.slots);
+        assert_eq!(allocation.prefix_len, 0);
+        cache.free(allocation.blocks.clone(), allocation.allocation_id);
+
+        let allocation = cache.allocate(8, Some(Arc::new(vec![0, 1, 2, 3]))).unwrap();
+        assert_eq!(allocation.blocks, vec![4, 5, 6, 7, 8, 9, 10, 11]);
+        assert_eq!(allocation.prefix_len, 4);
+    }
+
+    #[test]
+    fn allocator_collects_older_prefixes_first() {
+        let mut cache = RadixAllocator::new(1, 7, None);
+        let allocation1 = cache.allocate(4, Some(Arc::new(vec![0, 1, 2, 3]))).unwrap();
+        assert_eq!(allocation1.blocks, vec![3, 4, 5, 6]);
+        assert_eq!(allocation1.prefix_len, 0);
+
+        let allocation2 = cache.allocate(2, Some(Arc::new(vec![4, 5]))).unwrap();
+        assert_eq!(allocation2.blocks, vec![1, 2]);
+        assert_eq!(allocation2.prefix_len, 0);
+
+        cache.free(allocation1.blocks.clone(), allocation1.allocation_id);
+        cache.free(allocation2.blocks.clone(), allocation2.allocation_id);
+
+        // We should get the blocks of the first allocation, since they are more recent.
+        let allocation3 = cache.allocate(4, Some(Arc::new(vec![6, 7, 8, 9]))).unwrap();
+        assert_eq!(allocation3.blocks, vec![3, 4, 5, 6]);
+        assert_eq!(allocation3.prefix_len, 0);
+    }
+
+    #[test]
+    fn allocator_frees_fully_overlapping_prefills() {
+        let mut cache = RadixAllocator::new(1, 10, None);
+        let allocation1 = cache.allocate(4, Some(Arc::new(vec![0, 1, 2, 3]))).unwrap();
+        let allocation2 = cache.allocate(4, Some(Arc::new(vec![0, 1, 2, 3]))).unwrap();
+
+        cache.free(allocation2.blocks.clone(), allocation2.allocation_id);
+        cache.free(allocation1.blocks.clone(), allocation1.allocation_id);
+
+        let allocation3 = cache.allocate(4, Some(Arc::new(vec![0, 1, 2, 3]))).unwrap();
+        assert_eq!(allocation3.prefix_len, 4);
+
+        // 10 blocks, of which 1 reserved for health checks, 4 for the cached blocks.
+        assert_eq!(cache.free_blocks.len(), 5);
+    }
+
+    #[test]
+    fn allocator_frees_partially_overlapping_prefills() {
+        let mut cache = RadixAllocator::new(1, 20, None);
+        let allocation1 = cache.allocate(4, Some(Arc::new(vec![0, 1]))).unwrap();
+        assert_eq!(allocation1.blocks, vec![16, 17, 18, 19]);
+        assert_eq!(allocation1.prefix_len, 0);
+
+        cache.free(allocation1.blocks.clone(), allocation1.allocation_id);
+
+        let allocation2 = cache
+            .allocate(8, Some(Arc::new(vec![0, 1, 2, 3, 4, 5])))
+            .unwrap();
+        assert_eq!(allocation2.blocks, vec![16, 17, 12, 13, 14, 15, 18, 19]);
+        assert_eq!(allocation2.prefix_len, 2);
+
+        let allocation3 = cache
+            .allocate(8, Some(Arc::new(vec![0, 1, 2, 3, 6, 7])))
+            .unwrap();
+        assert_eq!(allocation3.blocks, vec![16, 17, 6, 7, 8, 9, 10, 11]);
+        assert_eq!(allocation3.prefix_len, 2);
+
+        cache.free(allocation3.blocks.clone(), allocation3.allocation_id);
+        cache.free(allocation2.blocks.clone(), allocation2.allocation_id);
+
+        // 20 blocks, of which 1 reserved for health checks, 6 for allocation3, 2 for allocation2.
+        assert_eq!(cache.free_blocks.len(), 11);
+
+        let allocation4 = cache
+            .allocate(6, Some(Arc::new(vec![0, 1, 2, 3, 4, 5])))
+            .unwrap();
+        assert_eq!(allocation4.blocks, vec![16, 17, 6, 7, 14, 15]);
+        assert_eq!(allocation4.prefix_len, 6);
+        assert_eq!(cache.free_blocks.len(), 11);
+
+        let allocation5 = cache
+            .allocate(6, Some(Arc::new(vec![0, 1, 2, 3, 6, 7])))
+            .unwrap();
+        assert_eq!(allocation5.blocks, vec![16, 17, 6, 7, 8, 9]);
+        assert_eq!(allocation5.prefix_len, 6);
+        assert_eq!(cache.free_blocks.len(), 11);
+    }
+
+    #[test]
+    fn trie_insertions_have_correct_prefix_len() {
+        let mut trie = RadixTrie::new(1);
+
+        assert_eq!(trie.insert(&[0, 1, 2], &[0, 1, 2]).unwrap(), 0);
+
+        // Already exists.
+        assert_eq!(trie.insert(&[0, 1, 2], &[0, 1, 2]).unwrap(), 3);
+
+        // Completely new at root-level
+        assert_eq!(trie.insert(&[1, 2, 3], &[1, 2, 3]).unwrap(), 0);
+
+        // Contains full prefix, but longer.
+        assert_eq!(trie.insert(&[0, 1, 2, 3, 4], &[0, 1, 2, 3, 4]).unwrap(), 3);
+
+        // Shares partial prefix, we need a split.
+        assert_eq!(
+            trie.insert(&[0, 1, 2, 3, 5, 6, 7], &[0, 1, 2, 3, 5, 6, 7])
+                .unwrap(),
+            4
+        );
+    }
+
+    #[test]
+    fn trie_insertions_block_size() {
+        let mut trie = RadixTrie::new(2);
+
+        assert_eq!(trie.insert(&[0, 1, 2, 3], &[0, 1]).unwrap(), 0);
+
+        // Already exists.
+        // But needs to be block_size aligned
+        assert_eq!(trie.insert(&[0, 1, 2, 3], &[0, 1]).unwrap(), 4);
+
+        // Completely new at root-level
+        assert_eq!(trie.insert(&[1, 2, 3, 4], &[1, 2]).unwrap(), 0);
+
+        // Contains full prefix, but longer.
+        assert_eq!(trie.insert(&[0, 1, 2, 3, 4, 5], &[0, 1, 2]).unwrap(), 4);
+
+        // Shares partial prefix, we need a split.
+        assert_eq!(
+            trie.insert(&[0, 1, 3, 4, 5, 6, 7, 8], &[0, 1, 2, 3])
+                .unwrap(),
+            2
+        );
+    }
+
+    #[test]
+    fn trie_get_returns_correct_blocks() {
+        let mut trie = RadixTrie::new(1);
+        trie.insert(&[0, 1, 2], &[0, 1, 2]).unwrap();
+        trie.insert(&[1, 2, 3], &[1, 2, 3]).unwrap();
+        trie.insert(&[0, 1, 2, 3, 4], &[0, 1, 2, 3, 4]).unwrap();
+        trie.insert(&[0, 1, 2, 3, 5, 6, 7], &[0, 1, 2, 3, 5, 6, 7])
+            .unwrap();
+
+        let mut blocks = Vec::new();
+        trie.find(&[0], &mut blocks);
+        assert_eq!(blocks, vec![0]);
+
+        blocks.clear();
+        trie.find(&[0, 1, 2], &mut blocks);
+        assert_eq!(blocks, vec![0, 1, 2]);
+
+        blocks.clear();
+        trie.find(&[1, 2, 3], &mut blocks);
+        assert_eq!(blocks, vec![1, 2, 3]);
+
+        blocks.clear();
+        trie.find(&[0, 1, 2, 3], &mut blocks);
+        assert_eq!(blocks, vec![0, 1, 2, 3]);
+
+        blocks.clear();
+        trie.find(&[0, 1, 2, 3, 4], &mut blocks);
+        assert_eq!(blocks, vec![0, 1, 2, 3, 4]);
+
+        blocks.clear();
+        trie.find(&[0, 1, 2, 3, 5], &mut blocks);
+        assert_eq!(blocks, vec![0, 1, 2, 3, 5]);
+    }
+
+    #[test]
+    fn trie_evict_removes_correct_blocks() {
+        let mut trie = RadixTrie::new(1);
+        trie.insert(&[0, 1, 2], &[0, 1, 2]).unwrap();
+        trie.insert(&[0, 1, 2, 3, 5, 6, 7], &[0, 1, 2, 3, 5, 6, 7])
+            .unwrap();
+        trie.insert(&[0, 1, 2, 3, 4], &[0, 1, 2, 3, 4]).unwrap();
+        trie.insert(&[1, 2, 3], &[1, 2, 3]).unwrap();
+
+        let mut blocks = Vec::new();
+
+        // Remove less than the leave blocks.
+        assert_eq!(trie.evict(1), vec![7]);
+        trie.find(&[0, 1, 2, 3, 5, 6, 7], &mut blocks);
+        assert_eq!(blocks, vec![0, 1, 2, 3, 5, 6]);
+
+        // Refresh other leaf.
+        trie.find(&[0, 1, 2, 3, 4], &mut blocks);
+        trie.find(&[1, 2, 3], &mut blocks);
+
+        // Remove the leave blocks exactly.
+        assert_eq!(trie.evict(2), vec![5, 6]);
+        blocks.clear();
+        trie.find(&[0, 1, 2, 3, 5, 6, 7], &mut blocks);
+        assert_eq!(blocks, vec![0, 1, 2, 3]);
+
+        trie.find(&[1, 2, 3], &mut blocks);
+
+        // Remove more than the leave blocks.
+        assert_eq!(trie.evict(3), vec![4, 3, 2]);
+        blocks.clear();
+        trie.find(&[0, 1, 2, 3, 4], &mut blocks);
+        assert_eq!(blocks, vec![0, 1]);
+
+        // Clear out the whole trie.
+        assert_eq!(trie.evict(10), vec![1, 2, 3, 0, 1]);
+    }
+}
diff --git a/benchmark/Cargo.toml b/benchmark/Cargo.toml
index 756460e0..23885da2 100644
--- a/benchmark/Cargo.toml
+++ b/benchmark/Cargo.toml
@@ -16,16 +16,15 @@ path = "src/main.rs"
 [dependencies]
 average = "0.14"
 clap = { version = "4.4.5", features = ["derive", "env"] }
-crossterm = "0.27"
 float-ord = "0.3.2"
 serde = {version = "1.0.188", features = ["derive"]}
 serde_json = "1.0"
 tabled = "0.14.0"
-text-generation-client = { path = "../router/client" }
+text-generation-client = { path = "../backends/client" }
 thiserror = "1.0.48"
 tokenizers = { workspace = true }
 tokio = { version = "1.32.0", features = ["rt", "rt-multi-thread", "parking_lot", "signal", "sync", "macros"] }
-tui = {package = "ratatui", version = "0.23", default-features = false, features = ["crossterm"]}
+ratatui = "0.28.1"
 tracing = "0.1.37"
 tracing-subscriber = { version = "0.3.17", features = ["json", "env-filter"] }
 hf-hub = { workspace = true }
diff --git a/benchmark/README.md b/benchmark/README.md
index 17a02a30..f4e0cb16 100644
--- a/benchmark/README.md
+++ b/benchmark/README.md
@@ -7,7 +7,7 @@
 </div>
 
 A lightweight benchmarking tool based inspired by [oha](https://github.com/hatoo/oha)
-and powered by [tui](https://github.com/tui-rs-revival/ratatui).
+and powered by [Ratatui](https://github.com/ratatui/ratatui).
 
 ## Install
 
diff --git a/benchmark/src/app.rs b/benchmark/src/app.rs
index 48ac976a..7e3aeaf9 100644
--- a/benchmark/src/app.rs
+++ b/benchmark/src/app.rs
@@ -1,16 +1,15 @@
 /// Inspired by https://github.com/hatoo/oha/blob/bb989ea3cd77727e7743e7daa60a19894bb5e901/src/monitor.rs
 use crate::generation::{Decode, Message, Prefill};
-use crossterm::event::{KeyCode, KeyEvent, KeyModifiers};
-use text_generation_client::ClientError;
-use tokio::sync::mpsc;
-use tui::backend::Backend;
-use tui::layout::{Alignment, Constraint, Direction, Layout};
-use tui::style::{Color, Modifier, Style};
-use tui::text::{Line, Span};
-use tui::widgets::{
+use ratatui::crossterm::event::{KeyCode, KeyEvent, KeyModifiers};
+use ratatui::layout::{Alignment, Constraint, Direction, Layout};
+use ratatui::style::{Color, Modifier, Style};
+use ratatui::text::{Line, Span};
+use ratatui::widgets::{
     Axis, BarChart, Block, Borders, Chart, Dataset, Gauge, GraphType, Paragraph, Tabs,
 };
-use tui::{symbols, Frame};
+use ratatui::{symbols, Frame};
+use text_generation_client::ClientError;
+use tokio::sync::mpsc;
 
 /// TUI powered App
 pub(crate) struct App {
@@ -153,7 +152,7 @@ impl App {
     }
 
     /// Render frame
-    pub fn render<B: Backend>(&mut self, f: &mut Frame<'_, B>) {
+    pub fn render(&mut self, f: &mut Frame) {
         let batch_progress =
             (self.completed_batch as f64 / self.data.batch_size.len() as f64).clamp(0.0, 1.0);
         let run_progress =
@@ -172,7 +171,7 @@ impl App {
                 ]
                 .as_ref(),
             )
-            .split(f.size());
+            .split(f.area());
 
         // Top row horizontal layout
         let top = Layout::default()
@@ -239,7 +238,7 @@ impl App {
         f.render_widget(helper, row5[0]);
 
         // Batch tabs
-        let titles = self
+        let titles: Vec<Line> = self
             .data
             .batch_size
             .iter()
@@ -497,7 +496,7 @@ fn statis_spans<'a>(data: &[f64], unit: &'static str) -> Vec<Line<'a>> {
                 "Lowest:  {:.2} {unit}",
                 data.iter()
                     .min_by(|a, b| a.total_cmp(b))
-                    .unwrap_or(&std::f64::NAN)
+                    .unwrap_or(&f64::NAN)
             ),
             Style::default().fg(Color::Reset),
         )]),
@@ -506,7 +505,7 @@ fn statis_spans<'a>(data: &[f64], unit: &'static str) -> Vec<Line<'a>> {
                 "Highest: {:.2} {unit}",
                 data.iter()
                     .max_by(|a, b| a.total_cmp(b))
-                    .unwrap_or(&std::f64::NAN)
+                    .unwrap_or(&f64::NAN)
             ),
             Style::default().fg(Color::Reset),
         )]),
@@ -555,17 +554,17 @@ fn latency_throughput_chart<'a>(
     let min_latency: f64 = *latency_iter
         .clone()
         .min_by(|a, b| a.total_cmp(b))
-        .unwrap_or(&std::f64::NAN);
+        .unwrap_or(&f64::NAN);
     let max_latency: f64 = *latency_iter
         .max_by(|a, b| a.total_cmp(b))
-        .unwrap_or(&std::f64::NAN);
+        .unwrap_or(&f64::NAN);
     let min_throughput: f64 = *throughput_iter
         .clone()
         .min_by(|a, b| a.total_cmp(b))
-        .unwrap_or(&std::f64::NAN);
+        .unwrap_or(&f64::NAN);
     let max_throughput: f64 = *throughput_iter
         .max_by(|a, b| a.total_cmp(b))
-        .unwrap_or(&std::f64::NAN);
+        .unwrap_or(&f64::NAN);
 
     // Char min max values
     let min_x = if zoom {
diff --git a/benchmark/src/event.rs b/benchmark/src/event.rs
index 07482aed..d3f10fb6 100644
--- a/benchmark/src/event.rs
+++ b/benchmark/src/event.rs
@@ -1,5 +1,5 @@
 /// Inspired by https://github.com/orhun/rust-tui-template/blob/472aa515119d4c94903eac12d9784417281dc7f5/src/event.rs
-use crossterm::event;
+use ratatui::crossterm::event;
 use std::time::{Duration, Instant};
 use tokio::sync::{broadcast, mpsc};
 
diff --git a/benchmark/src/generation.rs b/benchmark/src/generation.rs
index b2766d0c..789c7b51 100644
--- a/benchmark/src/generation.rs
+++ b/benchmark/src/generation.rs
@@ -1,8 +1,9 @@
 use std::time::{Duration, Instant};
-use text_generation_client::{
-    Batch, CachedBatch, ClientError, NextTokenChooserParameters, Request, ShardedClient,
+use text_generation_client::v3::{
+    Batch, CachedBatch, NextTokenChooserParameters, Request, ShardedClient,
     StoppingCriteriaParameters,
 };
+use text_generation_client::{Chunk, ClientError, Input};
 use tokenizers::{Tokenizer, TruncationDirection};
 use tokio::sync::{broadcast, mpsc};
 
@@ -142,8 +143,12 @@ async fn prefill(
         .map(|id| Request {
             id: id.into(),
             prefill_logprobs: false,
+            input_chunks: Some(Input {
+                chunks: vec![Chunk::Text(sequence.clone()).into()],
+            }),
             inputs: sequence.clone(),
             truncate: sequence_length,
+            add_special_tokens: true,
             parameters: Some(parameters.clone()),
             stopping_parameters: Some(StoppingCriteriaParameters {
                 max_new_tokens: decode_length,
@@ -151,6 +156,10 @@ async fn prefill(
                 ignore_eos_token: true, // Will not stop even if a eos token is generated
             }),
             top_n_tokens: top_n_tokens.unwrap_or(0),
+            blocks: vec![],
+            slots: vec![],
+            prefix_len: 0,
+            adapter_id: None,
         })
         .collect();
 
@@ -159,15 +168,13 @@ async fn prefill(
         requests,
         size: batch_size,
         max_tokens: batch_size * (sequence_length + decode_length),
+        max_blocks: 0,
     };
 
     // Run prefill
     let start_time = Instant::now();
-
     let (_, decode_batch, _) = client.prefill(batch.clone()).await?;
 
-    let (_, decode_batch, _) = client.decode(vec![decode_batch.clone().unwrap()]).await?;
-
     // Get latency
     let latency = start_time.elapsed();
 
@@ -183,12 +190,11 @@ async fn prefill(
     };
 
     Ok((step, decode_batch))
-
 }
 
 /// Run a full decode
 async fn decode(batch: CachedBatch, client: &mut ShardedClient) -> Result<Decode, ClientError> {
-    let mut decode_length = 1; // 1 decode step was already scheduled in prefill with speculative scheduling
+    let mut decode_length = 0;
     let batch_size = batch.size;
 
     let start_time = Instant::now();
diff --git a/benchmark/src/lib.rs b/benchmark/src/lib.rs
index 638c6514..bb4b6a77 100644
--- a/benchmark/src/lib.rs
+++ b/benchmark/src/lib.rs
@@ -6,13 +6,13 @@ mod utils;
 
 use crate::app::App;
 use crate::event::Event;
-use crossterm::ExecutableCommand;
+use ratatui::backend::CrosstermBackend;
+use ratatui::crossterm::ExecutableCommand;
+use ratatui::Terminal;
 use std::io;
-use text_generation_client::{GrammarType, NextTokenChooserParameters, ShardedClient};
+use text_generation_client::v3::{GrammarType, NextTokenChooserParameters, ShardedClient};
 use tokenizers::Tokenizer;
 use tokio::sync::{broadcast, mpsc};
-use tui::backend::CrosstermBackend;
-use tui::Terminal;
 
 /// Run benchmarking app
 #[allow(clippy::too_many_arguments)]
@@ -50,9 +50,9 @@ pub async fn run(
     };
 
     // Initialize terminal properties
-    crossterm::terminal::enable_raw_mode()?;
-    io::stdout().execute(crossterm::terminal::EnterAlternateScreen)?;
-    io::stdout().execute(crossterm::cursor::Hide)?;
+    ratatui::crossterm::terminal::enable_raw_mode()?;
+    io::stdout().execute(ratatui::crossterm::terminal::EnterAlternateScreen)?;
+    io::stdout().execute(ratatui::crossterm::cursor::Hide)?;
 
     // Initialize terminal
     let mut terminal = {
@@ -128,9 +128,9 @@ pub async fn run(
     let _ = shutdown_guard_receiver.recv().await;
 
     // Revert terminal to original view
-    io::stdout().execute(crossterm::terminal::LeaveAlternateScreen)?;
-    crossterm::terminal::disable_raw_mode()?;
-    io::stdout().execute(crossterm::cursor::Show)?;
+    io::stdout().execute(ratatui::crossterm::terminal::LeaveAlternateScreen)?;
+    ratatui::crossterm::terminal::disable_raw_mode()?;
+    io::stdout().execute(ratatui::crossterm::cursor::Show)?;
 
     let parameters_table = table::parameters_table(
         tokenizer_name,
diff --git a/benchmark/src/main.rs b/benchmark/src/main.rs
index 935808b6..2ee3d7c5 100644
--- a/benchmark/src/main.rs
+++ b/benchmark/src/main.rs
@@ -4,7 +4,7 @@
 /// and: https://github.com/orhun/rust-tui-template
 use clap::Parser;
 use std::path::Path;
-use text_generation_client::ShardedClient;
+use text_generation_client::v3::ShardedClient;
 use tokenizers::{FromPretrainedParameters, Tokenizer};
 use tracing_subscriber::layer::SubscriberExt;
 use tracing_subscriber::util::SubscriberInitExt;
@@ -51,7 +51,7 @@ struct Args {
     runs: usize,
 
     /// Number of warmup cycles
-    #[clap(default_value = "3", short, long, env)]
+    #[clap(default_value = "1", short, long, env)]
     warmups: usize,
 
     /// The location of the grpc socket. This benchmark tool bypasses the router
@@ -155,7 +155,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
             // We need to download it outside of the Tokio runtime
             let params = FromPretrainedParameters {
                 revision,
-                token: auth_token,
+                auth_token,
                 ..Default::default()
             };
             Tokenizer::from_pretrained(tokenizer_name.clone(), Some(params)).unwrap()
diff --git a/benchmark/src/table.rs b/benchmark/src/table.rs
index e18d7310..1585a25f 100644
--- a/benchmark/src/table.rs
+++ b/benchmark/src/table.rs
@@ -156,17 +156,17 @@ fn avg_min_max(data: &[f64]) -> (f64, f64, f64) {
     let min = data
         .iter()
         .min_by(|a, b| a.total_cmp(b))
-        .unwrap_or(&std::f64::NAN);
+        .unwrap_or(&f64::NAN);
     let max = data
         .iter()
         .max_by(|a, b| a.total_cmp(b))
-        .unwrap_or(&std::f64::NAN);
+        .unwrap_or(&f64::NAN);
     (average, *min, *max)
 }
 
 fn px(data: &[f64], p: u32) -> f64 {
     let i = (f64::from(p) / 100.0 * data.len() as f64) as usize;
-    *data.get(i).unwrap_or(&std::f64::NAN)
+    *data.get(i).unwrap_or(&f64::NAN)
 }
 
 fn format_value(value: f64, unit: &'static str) -> String {
diff --git a/benchmark/src/utils.rs b/benchmark/src/utils.rs
index d096d655..20469991 100644
--- a/benchmark/src/utils.rs
+++ b/benchmark/src/utils.rs
@@ -37,7 +37,7 @@ pub(crate) fn percentiles(values: &[f64], pecents: &[i32]) -> BTreeMap<String, f
         .iter()
         .map(|&p| {
             let i = (f64::from(p) / 100.0 * values.len() as f64) as usize;
-            (format!("p{p}"), *values.get(i).unwrap_or(&std::f64::NAN))
+            (format!("p{p}"), *values.get(i).unwrap_or(&f64::NAN))
         })
         .collect()
 }
diff --git a/clients/python/README.md b/clients/python/README.md
index bf37508e..88239aa1 100644
--- a/clients/python/README.md
+++ b/clients/python/README.md
@@ -1,3 +1,6 @@
+# Legacy warning ⚠️
+The inference clients from [huggingface_hub](https://huggingface.co/docs/huggingface_hub/guides/inference) are recommended over `text_generation`.
+
 # Text Generation
 
 The Hugging Face Text Generation Python library provides a convenient way of interfacing with a
diff --git a/clients/python/text_generation/__init__.py b/clients/python/text_generation/__init__.py
index a8e67071..ca783dcd 100644
--- a/clients/python/text_generation/__init__.py
+++ b/clients/python/text_generation/__init__.py
@@ -12,12 +12,22 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-__version__ = "0.6.0"
+__version__ = "0.7.0"
 
 DEPRECATION_WARNING = (
     "`text_generation` clients are deprecated and will be removed in the near future. "
     "Please use the `InferenceClient` from the `huggingface_hub` package instead."
 )
 
-from text_generation.client import Client, AsyncClient
-from text_generation.inference_api import InferenceAPIClient, InferenceAPIAsyncClient
+from text_generation.client import Client, AsyncClient  # noqa E402
+from text_generation.inference_api import (  # noqa E402
+    InferenceAPIClient,
+    InferenceAPIAsyncClient,
+)
+
+__all__ = [
+    "Client",
+    "AsyncClient",
+    "InferenceAPIClient",
+    "InferenceAPIAsyncClient",
+]
diff --git a/clients/python/text_generation/client.py b/clients/python/text_generation/client.py
index 12966747..45301b63 100644
--- a/clients/python/text_generation/client.py
+++ b/clients/python/text_generation/client.py
@@ -757,7 +757,12 @@ class AsyncClient:
                         continue
                     payload = byte_payload.decode("utf-8")
                     if payload.startswith("data:"):
-                        json_payload = json.loads(payload.lstrip("data:").rstrip("\n"))
+                        payload_data = (
+                            payload.lstrip("data:").rstrip("\n").removeprefix(" ")
+                        )
+                        if payload_data == "[DONE]":
+                            break
+                        json_payload = json.loads(payload_data)
                         try:
                             response = ChatCompletionChunk(**json_payload)
                             yield response
diff --git a/clients/python/text_generation/inference_api.py b/clients/python/text_generation/inference_api.py
index 93b0de8d..b3b98ed2 100644
--- a/clients/python/text_generation/inference_api.py
+++ b/clients/python/text_generation/inference_api.py
@@ -21,7 +21,7 @@ def deployed_models(headers: Optional[Dict] = None) -> List[DeployedModel]:
         List[DeployedModel]: list of all currently deployed models
     """
     resp = requests.get(
-        f"https://api-inference.huggingface.co/framework/text-generation-inference",
+        "https://api-inference.huggingface.co/framework/text-generation-inference",
         headers=headers,
         timeout=5,
     )
diff --git a/clients/python/text_generation/types.py b/clients/python/text_generation/types.py
index eb872ee6..1085075e 100644
--- a/clients/python/text_generation/types.py
+++ b/clients/python/text_generation/types.py
@@ -1,5 +1,5 @@
 from enum import Enum
-from pydantic import BaseModel, field_validator
+from pydantic import BaseModel, field_validator, ConfigDict
 from typing import Optional, List, Union, Any
 
 from text_generation.errors import ValidationError
@@ -28,11 +28,17 @@ class ToolCall(BaseModel):
     function: dict
 
 
+class Chunk(BaseModel):
+    type: str
+    text: Optional[str] = None
+    image_url: Any = None
+
+
 class Message(BaseModel):
     # Role of the message sender
     role: str
     # Content of the message
-    content: Optional[str] = None
+    content: Optional[Union[str, List[Chunk]]] = None
     # Optional name of the message sender
     name: Optional[str] = None
     # Tool calls associated with the chat completion
@@ -61,7 +67,7 @@ class ChoiceDeltaToolCall(BaseModel):
 class ChoiceDelta(BaseModel):
     role: str
     content: Optional[str] = None
-    tool_calls: Optional[ChoiceDeltaToolCall]
+    tool_calls: Optional[ChoiceDeltaToolCall] = None
 
 
 class Choice(BaseModel):
@@ -168,7 +174,7 @@ class ChatCompletionComplete(BaseModel):
     # Log probabilities for the chat completion
     logprobs: Optional[Any]
     # Reason for completion
-    finish_reason: str
+    finish_reason: Optional[str]
     # Usage details of the chat completion
     usage: Optional[Any] = None
 
@@ -191,6 +197,7 @@ class ChatCompletionChunk(BaseModel):
     model: str
     system_fingerprint: str
     choices: List[Choice]
+    usage: Optional[Any] = None
 
 
 class Parameters(BaseModel):
@@ -452,5 +459,9 @@ class StreamResponse(BaseModel):
 
 # Inference API currently deployed model
 class DeployedModel(BaseModel):
+    # Disable warning for use of `model_` prefix in `model_id`. Be mindful about adding members
+    # with model_ prefixes, since this disables guardrails for colliding fields:
+    # https://github.com/pydantic/pydantic/issues/9177
+    model_config = ConfigDict(protected_namespaces=())
     model_id: str
     sha: str
diff --git a/docs/README.md b/docs/README.md
new file mode 100644
index 00000000..fb2ff198
--- /dev/null
+++ b/docs/README.md
@@ -0,0 +1,10 @@
+Documentation available at: https://huggingface.co/docs/text-generation-inference
+
+## Release
+
+When making a release, please update the latest version in the documentation with:
+```
+export OLD_VERSION="2\.0\.3"
+export NEW_VERSION="2\.0\.4"
+find . -name '*.md' -exec sed -i -e "s/$OLD_VERSION/$NEW_VERSION/g" {} \;
+```
diff --git a/docs/openapi.json b/docs/openapi.json
index 79c3b80f..5854bcdd 100644
--- a/docs/openapi.json
+++ b/docs/openapi.json
@@ -10,7 +10,7 @@
       "name": "Apache 2.0",
       "url": "https://www.apache.org/licenses/LICENSE-2.0"
     },
-    "version": "2.0.1"
+    "version": "2.3.1"
   },
   "paths": {
     "/": {
@@ -19,7 +19,6 @@
           "Text Generation Inference"
         ],
         "summary": "Generate tokens if `stream == false` or a stream of token if `stream == true`",
-        "description": "Generate tokens if `stream == false` or a stream of token if `stream == true`",
         "operationId": "compat_generate",
         "requestBody": {
           "content": {
@@ -108,7 +107,6 @@
           "Text Generation Inference"
         ],
         "summary": "Generate tokens",
-        "description": "Generate tokens",
         "operationId": "generate",
         "requestBody": {
           "content": {
@@ -192,7 +190,6 @@
           "Text Generation Inference"
         ],
         "summary": "Generate a stream of token using Server-Sent Events",
-        "description": "Generate a stream of token using Server-Sent Events",
         "operationId": "generate_stream",
         "requestBody": {
           "content": {
@@ -276,7 +273,6 @@
           "Text Generation Inference"
         ],
         "summary": "Health check method",
-        "description": "Health check method",
         "operationId": "health",
         "responses": {
           "200": {
@@ -305,7 +301,6 @@
           "Text Generation Inference"
         ],
         "summary": "Text Generation Inference endpoint info",
-        "description": "Text Generation Inference endpoint info",
         "operationId": "get_model_info",
         "responses": {
           "200": {
@@ -327,7 +322,6 @@
           "Text Generation Inference"
         ],
         "summary": "Prometheus metrics scrape endpoint",
-        "description": "Prometheus metrics scrape endpoint",
         "operationId": "metrics",
         "responses": {
           "200": {
@@ -349,7 +343,6 @@
           "Text Generation Inference"
         ],
         "summary": "Tokenize inputs",
-        "description": "Tokenize inputs",
         "operationId": "tokenize",
         "requestBody": {
           "content": {
@@ -394,7 +387,6 @@
           "Text Generation Inference"
         ],
         "summary": "Generate tokens",
-        "description": "Generate tokens",
         "operationId": "chat_completions",
         "requestBody": {
           "content": {
@@ -483,7 +475,6 @@
           "Text Generation Inference"
         ],
         "summary": "Generate tokens",
-        "description": "Generate tokens",
         "operationId": "completions",
         "requestBody": {
           "content": {
@@ -501,12 +492,12 @@
             "content": {
               "application/json": {
                 "schema": {
-                  "$ref": "#/components/schemas/Completion"
+                  "$ref": "#/components/schemas/CompletionFinal"
                 }
               },
               "text/event-stream": {
                 "schema": {
-                  "$ref": "#/components/schemas/CompletionCompleteChunk"
+                  "$ref": "#/components/schemas/Chunk"
                 }
               }
             }
@@ -565,6 +556,37 @@
           }
         }
       }
+    },
+    "/v1/models": {
+      "get": {
+        "tags": [
+          "Text Generation Inference"
+        ],
+        "summary": "Get model info",
+        "operationId": "openai_get_model_info",
+        "responses": {
+          "200": {
+            "description": "Served model info",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/ModelInfo"
+                }
+              }
+            }
+          },
+          "404": {
+            "description": "Model not found",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/ErrorResponse"
+                }
+              }
+            }
+          }
+        }
+      }
     }
   },
   "components": {
@@ -626,7 +648,6 @@
         "type": "object",
         "required": [
           "id",
-          "object",
           "created",
           "model",
           "system_fingerprint",
@@ -653,9 +674,6 @@
             "type": "string",
             "example": "mistralai/Mistral-7B-Instruct-v0.2"
           },
-          "object": {
-            "type": "string"
-          },
           "system_fingerprint": {
             "type": "string"
           },
@@ -697,7 +715,6 @@
         "type": "object",
         "required": [
           "id",
-          "object",
           "created",
           "model",
           "system_fingerprint",
@@ -723,11 +740,16 @@
             "type": "string",
             "example": "mistralai/Mistral-7B-Instruct-v0.2"
           },
-          "object": {
-            "type": "string"
-          },
           "system_fingerprint": {
             "type": "string"
+          },
+          "usage": {
+            "allOf": [
+              {
+                "$ref": "#/components/schemas/Usage"
+              }
+            ],
+            "nullable": true
           }
         }
       },
@@ -756,34 +778,19 @@
             "nullable": true
           },
           "message": {
-            "$ref": "#/components/schemas/Message"
+            "$ref": "#/components/schemas/OutputMessage"
           }
         }
       },
       "ChatCompletionDelta": {
-        "type": "object",
-        "required": [
-          "role"
-        ],
-        "properties": {
-          "content": {
-            "type": "string",
-            "example": "What is Deep Learning?",
-            "nullable": true
+        "oneOf": [
+          {
+            "$ref": "#/components/schemas/TextMessage"
           },
-          "role": {
-            "type": "string",
-            "example": "user"
-          },
-          "tool_calls": {
-            "allOf": [
-              {
-                "$ref": "#/components/schemas/DeltaToolCall"
-              }
-            ],
-            "nullable": true
+          {
+            "$ref": "#/components/schemas/ToolCallDelta"
           }
-        }
+        ]
       },
       "ChatCompletionLogprob": {
         "type": "object",
@@ -841,7 +848,6 @@
       "ChatRequest": {
         "type": "object",
         "required": [
-          "model",
           "messages"
         ],
         "properties": {
@@ -852,6 +858,13 @@
             "example": "1.0",
             "nullable": true
           },
+          "guideline": {
+            "type": "string",
+            "description": "A guideline to be used in the chat_template",
+            "default": "null",
+            "example": "null",
+            "nullable": true
+          },
           "logit_bias": {
             "type": "array",
             "items": {
@@ -886,7 +899,8 @@
           "model": {
             "type": "string",
             "description": "[UNUSED] ID of the model to use. See the model endpoint compatibility table for details on which models work with the Chat API.",
-            "example": "mistralai/Mistral-7B-Instruct-v0.2"
+            "example": "mistralai/Mistral-7B-Instruct-v0.2",
+            "nullable": true
           },
           "n": {
             "type": "integer",
@@ -903,6 +917,15 @@
             "example": 0.1,
             "nullable": true
           },
+          "response_format": {
+            "allOf": [
+              {
+                "$ref": "#/components/schemas/GrammarType"
+              }
+            ],
+            "default": "null",
+            "nullable": true
+          },
           "seed": {
             "type": "integer",
             "format": "int64",
@@ -922,6 +945,14 @@
           "stream": {
             "type": "boolean"
           },
+          "stream_options": {
+            "allOf": [
+              {
+                "$ref": "#/components/schemas/StreamOptions"
+              }
+            ],
+            "nullable": true
+          },
           "temperature": {
             "type": "number",
             "format": "float",
@@ -932,7 +963,7 @@
           "tool_choice": {
             "allOf": [
               {
-                "$ref": "#/components/schemas/ToolType"
+                "$ref": "#/components/schemas/ToolChoice"
               }
             ],
             "nullable": true
@@ -940,7 +971,7 @@
           "tool_prompt": {
             "type": "string",
             "description": "A prompt to be appended before the tools",
-            "example": "\"You will be presented with a JSON schema representing a set of tools.\nIf the user request lacks of sufficient information to make a precise tool selection: Do not invent any tool's properties, instead notify with an error message.\n\nJSON Schema:\n\"",
+            "example": "Given the functions available, please respond with a JSON for a function call with its proper arguments that best answers the given prompt. Respond in the format {name: function name, parameters: dictionary of argument name and its value}.Do not use variables.",
             "nullable": true
           },
           "tools": {
@@ -969,6 +1000,38 @@
           }
         }
       },
+      "Chunk": {
+        "type": "object",
+        "required": [
+          "id",
+          "created",
+          "choices",
+          "model",
+          "system_fingerprint"
+        ],
+        "properties": {
+          "choices": {
+            "type": "array",
+            "items": {
+              "$ref": "#/components/schemas/CompletionComplete"
+            }
+          },
+          "created": {
+            "type": "integer",
+            "format": "int64",
+            "minimum": 0
+          },
+          "id": {
+            "type": "string"
+          },
+          "model": {
+            "type": "string"
+          },
+          "system_fingerprint": {
+            "type": "string"
+          }
+        }
+      },
       "CompatGenerateRequest": {
         "type": "object",
         "required": [
@@ -988,6 +1051,55 @@
           }
         }
       },
+      "Completion": {
+        "oneOf": [
+          {
+            "allOf": [
+              {
+                "$ref": "#/components/schemas/Chunk"
+              },
+              {
+                "type": "object",
+                "required": [
+                  "object"
+                ],
+                "properties": {
+                  "object": {
+                    "type": "string",
+                    "enum": [
+                      "text_completion"
+                    ]
+                  }
+                }
+              }
+            ]
+          },
+          {
+            "allOf": [
+              {
+                "$ref": "#/components/schemas/CompletionFinal"
+              },
+              {
+                "type": "object",
+                "required": [
+                  "object"
+                ],
+                "properties": {
+                  "object": {
+                    "type": "string",
+                    "enum": [
+                      "text_completion"
+                    ]
+                  }
+                }
+              }
+            ]
+          }
+        ],
+        "discriminator": {
+          "propertyName": "object"
+        }
+      },
       "CompletionComplete": {
         "type": "object",
         "required": [
@@ -1017,15 +1129,15 @@
           }
         }
       },
-      "CompletionCompleteChunk": {
+      "CompletionFinal": {
         "type": "object",
         "required": [
           "id",
-          "object",
           "created",
-          "choices",
           "model",
-          "system_fingerprint"
+          "system_fingerprint",
+          "choices",
+          "usage"
         ],
         "properties": {
           "choices": {
@@ -1037,26 +1149,27 @@
           "created": {
             "type": "integer",
             "format": "int64",
+            "example": "1706270835",
             "minimum": 0
           },
           "id": {
             "type": "string"
           },
           "model": {
-            "type": "string"
-          },
-          "object": {
-            "type": "string"
+            "type": "string",
+            "example": "mistralai/Mistral-7B-Instruct-v0.2"
           },
           "system_fingerprint": {
             "type": "string"
+          },
+          "usage": {
+            "$ref": "#/components/schemas/Usage"
           }
         }
       },
       "CompletionRequest": {
         "type": "object",
         "required": [
-          "model",
           "prompt"
         ],
         "properties": {
@@ -1078,15 +1191,11 @@
           "model": {
             "type": "string",
             "description": "UNUSED\nID of the model to use. See the model endpoint compatibility table for details on which models work with the Chat API.",
-            "example": "mistralai/Mistral-7B-Instruct-v0.2"
+            "example": "mistralai/Mistral-7B-Instruct-v0.2",
+            "nullable": true
           },
           "prompt": {
-            "type": "array",
-            "items": {
-              "type": "string"
-            },
-            "description": "The prompt to generate completions for.",
-            "example": "What is Deep Learning?"
+            "$ref": "#/components/schemas/Prompt"
           },
           "repetition_penalty": {
             "type": "number",
@@ -1100,6 +1209,15 @@
             "nullable": true,
             "minimum": 0
           },
+          "stop": {
+            "type": "array",
+            "items": {
+              "type": "string"
+            },
+            "description": "Up to 4 sequences where the API will stop generating further tokens.",
+            "example": "null",
+            "nullable": true
+          },
           "stream": {
             "type": "boolean"
           },
@@ -1121,15 +1239,6 @@
             "description": "An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the\ntokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered.",
             "example": 0.95,
             "nullable": true
-          },
-          "stop": {
-            "type": "array",
-            "items": {
-              "type": "string"
-            },
-            "description": "Up to 4 sequences where the API will stop generating further tokens.",
-            "example": "null",
-            "nullable": true
           }
         }
       },
@@ -1269,11 +1378,30 @@
           }
         }
       },
+      "FunctionName": {
+        "type": "object",
+        "required": [
+          "name"
+        ],
+        "properties": {
+          "name": {
+            "type": "string"
+          }
+        }
+      },
       "GenerateParameters": {
         "type": "object",
         "properties": {
+          "adapter_id": {
+            "type": "string",
+            "description": "Lora adapter id",
+            "default": "null",
+            "example": "null",
+            "nullable": true
+          },
           "best_of": {
             "type": "integer",
+            "description": "Generate best_of sequences and return the one if the highest token logprobs.",
             "default": "null",
             "example": 1,
             "nullable": true,
@@ -1282,20 +1410,24 @@
           },
           "decoder_input_details": {
             "type": "boolean",
+            "description": "Whether to return decoder input token logprobs and ids.",
             "default": "false"
           },
           "details": {
             "type": "boolean",
+            "description": "Whether to return generation details.",
             "default": "true"
           },
           "do_sample": {
             "type": "boolean",
+            "description": "Activate logits sampling.",
             "default": "false",
             "example": true
           },
           "frequency_penalty": {
             "type": "number",
             "format": "float",
+            "description": "The parameter for frequency penalty. 1.0 means no penalty\nPenalize new tokens based on their existing frequency in the text so far,\ndecreasing the model's likelihood to repeat the same line verbatim.",
             "default": "null",
             "example": 0.1,
             "nullable": true,
@@ -1313,6 +1445,7 @@
           "max_new_tokens": {
             "type": "integer",
             "format": "int32",
+            "description": "Maximum number of tokens to generate.",
             "default": "100",
             "example": "20",
             "nullable": true,
@@ -1321,6 +1454,7 @@
           "repetition_penalty": {
             "type": "number",
             "format": "float",
+            "description": "The parameter for repetition penalty. 1.0 means no penalty.\nSee [this paper](https://arxiv.org/pdf/1909.05858.pdf) for more details.",
             "default": "null",
             "example": 1.03,
             "nullable": true,
@@ -1328,6 +1462,7 @@
           },
           "return_full_text": {
             "type": "boolean",
+            "description": "Whether to prepend the prompt to the generated text",
             "default": "null",
             "example": false,
             "nullable": true
@@ -1335,6 +1470,7 @@
           "seed": {
             "type": "integer",
             "format": "int64",
+            "description": "Random sampling seed.",
             "default": "null",
             "example": "null",
             "nullable": true,
@@ -1346,6 +1482,7 @@
             "items": {
               "type": "string"
             },
+            "description": "Stop generating tokens if a member of `stop` is generated.",
             "example": [
               "photographer"
             ],
@@ -1354,6 +1491,7 @@
           "temperature": {
             "type": "number",
             "format": "float",
+            "description": "The value used to module the logits distribution.",
             "default": "null",
             "example": 0.5,
             "nullable": true,
@@ -1362,6 +1500,7 @@
           "top_k": {
             "type": "integer",
             "format": "int32",
+            "description": "The number of highest probability vocabulary tokens to keep for top-k-filtering.",
             "default": "null",
             "example": 10,
             "nullable": true,
@@ -1370,6 +1509,7 @@
           "top_n_tokens": {
             "type": "integer",
             "format": "int32",
+            "description": "The number of highest probability vocabulary tokens to keep for top-n-filtering.",
             "default": "null",
             "example": 5,
             "nullable": true,
@@ -1379,6 +1519,7 @@
           "top_p": {
             "type": "number",
             "format": "float",
+            "description": "Top-p value for nucleus sampling.",
             "default": "null",
             "example": 0.95,
             "nullable": true,
@@ -1387,6 +1528,7 @@
           },
           "truncate": {
             "type": "integer",
+            "description": "Truncate inputs tokens to the given size.",
             "default": "null",
             "example": "null",
             "nullable": true,
@@ -1395,6 +1537,7 @@
           "typical_p": {
             "type": "number",
             "format": "float",
+            "description": "Typical Decoding mass\nSee [Typical Decoding for Natural Language Generation](https://arxiv.org/abs/2202.00666) for more information.",
             "default": "null",
             "example": 0.95,
             "nullable": true,
@@ -1403,6 +1546,7 @@
           },
           "watermark": {
             "type": "boolean",
+            "description": "Watermarking with [A Watermark for Large Language Models](https://arxiv.org/abs/2301.10226).",
             "default": "false",
             "example": true
           }
@@ -1490,18 +1634,14 @@
         "type": "object",
         "required": [
           "model_id",
-          "model_dtype",
-          "model_device_type",
           "max_concurrent_requests",
           "max_best_of",
           "max_stop_sequences",
-          "max_input_length",
+          "max_input_tokens",
           "max_total_tokens",
-          "waiting_served_ratio",
-          "max_batch_total_tokens",
-          "max_waiting_tokens",
           "validation_workers",
           "max_client_batch_size",
+          "router",
           "version"
         ],
         "properties": {
@@ -1510,18 +1650,6 @@
             "example": "null",
             "nullable": true
           },
-          "max_batch_size": {
-            "type": "integer",
-            "example": "null",
-            "nullable": true,
-            "minimum": 0
-          },
-          "max_batch_total_tokens": {
-            "type": "integer",
-            "format": "int32",
-            "example": "32000",
-            "minimum": 0
-          },
           "max_best_of": {
             "type": "integer",
             "example": "2",
@@ -1538,7 +1666,7 @@
             "example": "128",
             "minimum": 0
           },
-          "max_input_length": {
+          "max_input_tokens": {
             "type": "integer",
             "example": "1024",
             "minimum": 0
@@ -1553,19 +1681,6 @@
             "example": "2048",
             "minimum": 0
           },
-          "max_waiting_tokens": {
-            "type": "integer",
-            "example": "20",
-            "minimum": 0
-          },
-          "model_device_type": {
-            "type": "string",
-            "example": "cuda"
-          },
-          "model_dtype": {
-            "type": "string",
-            "example": "torch.float16"
-          },
           "model_id": {
             "type": "string",
             "description": "Model info",
@@ -1581,6 +1696,11 @@
             "example": "e985a63cdc139290c5f700ff1929f0b5942cced2",
             "nullable": true
           },
+          "router": {
+            "type": "string",
+            "description": "Router Info",
+            "example": "text-generation-router"
+          },
           "sha": {
             "type": "string",
             "example": "null",
@@ -1593,26 +1713,19 @@
           },
           "version": {
             "type": "string",
-            "description": "Router Info",
             "example": "0.5.0"
-          },
-          "waiting_served_ratio": {
-            "type": "number",
-            "format": "float",
-            "example": "1.2"
           }
         }
       },
       "Message": {
         "type": "object",
         "required": [
-          "role"
+          "role",
+          "content"
         ],
         "properties": {
           "content": {
-            "type": "string",
-            "example": "My name is David and I",
-            "nullable": true
+            "$ref": "#/components/schemas/MessageContent"
           },
           "name": {
             "type": "string",
@@ -1622,16 +1735,104 @@
           "role": {
             "type": "string",
             "example": "user"
-          },
-          "tool_calls": {
-            "type": "array",
-            "items": {
-              "$ref": "#/components/schemas/ToolCall"
-            },
-            "nullable": true
           }
         }
       },
+      "MessageChunk": {
+        "oneOf": [
+          {
+            "type": "object",
+            "required": [
+              "text",
+              "type"
+            ],
+            "properties": {
+              "text": {
+                "type": "string"
+              },
+              "type": {
+                "type": "string",
+                "enum": [
+                  "text"
+                ]
+              }
+            }
+          },
+          {
+            "type": "object",
+            "required": [
+              "image_url",
+              "type"
+            ],
+            "properties": {
+              "image_url": {
+                "$ref": "#/components/schemas/Url"
+              },
+              "type": {
+                "type": "string",
+                "enum": [
+                  "image_url"
+                ]
+              }
+            }
+          }
+        ],
+        "discriminator": {
+          "propertyName": "type"
+        }
+      },
+      "MessageContent": {
+        "oneOf": [
+          {
+            "type": "string"
+          },
+          {
+            "type": "array",
+            "items": {
+              "$ref": "#/components/schemas/MessageChunk"
+            }
+          }
+        ]
+      },
+      "ModelInfo": {
+        "type": "object",
+        "required": [
+          "id",
+          "object",
+          "created",
+          "owned_by"
+        ],
+        "properties": {
+          "created": {
+            "type": "integer",
+            "format": "int64",
+            "example": 1686935002,
+            "minimum": 0
+          },
+          "id": {
+            "type": "string",
+            "example": "gpt2"
+          },
+          "object": {
+            "type": "string",
+            "example": "model"
+          },
+          "owned_by": {
+            "type": "string",
+            "example": "openai"
+          }
+        }
+      },
+      "OutputMessage": {
+        "oneOf": [
+          {
+            "$ref": "#/components/schemas/TextMessage"
+          },
+          {
+            "$ref": "#/components/schemas/ToolCallMessage"
+          }
+        ]
+      },
       "PrefillToken": {
         "type": "object",
         "required": [
@@ -1658,6 +1859,12 @@
           }
         }
       },
+      "Prompt": {
+        "type": "array",
+        "items": {
+          "type": "string"
+        }
+      },
       "SimpleToken": {
         "type": "object",
         "required": [
@@ -1693,7 +1900,8 @@
         "type": "object",
         "required": [
           "finish_reason",
-          "generated_tokens"
+          "generated_tokens",
+          "input_length"
         ],
         "properties": {
           "finish_reason": {
@@ -1705,6 +1913,12 @@
             "example": 1,
             "minimum": 0
           },
+          "input_length": {
+            "type": "integer",
+            "format": "int32",
+            "example": 1,
+            "minimum": 0
+          },
           "seed": {
             "type": "integer",
             "format": "int64",
@@ -1714,6 +1928,19 @@
           }
         }
       },
+      "StreamOptions": {
+        "type": "object",
+        "required": [
+          "include_usage"
+        ],
+        "properties": {
+          "include_usage": {
+            "type": "boolean",
+            "description": "If set, an additional chunk will be streamed before the data: [DONE] message. The usage field on this chunk shows the token usage statistics for the entire request, and the choices field will always be an empty array. All other chunks will also include a usage field, but with a null value.",
+            "example": "true"
+          }
+        }
+      },
       "StreamResponse": {
         "type": "object",
         "required": [
@@ -1752,6 +1979,23 @@
           }
         }
       },
+      "TextMessage": {
+        "type": "object",
+        "required": [
+          "role",
+          "content"
+        ],
+        "properties": {
+          "content": {
+            "type": "string",
+            "example": "My name is David and I"
+          },
+          "role": {
+            "type": "string",
+            "example": "user"
+          }
+        }
+      },
       "Token": {
         "type": "object",
         "required": [
@@ -1817,36 +2061,95 @@
             "$ref": "#/components/schemas/FunctionDefinition"
           },
           "id": {
-            "type": "integer",
-            "format": "int32",
-            "minimum": 0
+            "type": "string"
           },
           "type": {
             "type": "string"
           }
         }
       },
+      "ToolCallDelta": {
+        "type": "object",
+        "required": [
+          "role",
+          "tool_calls"
+        ],
+        "properties": {
+          "role": {
+            "type": "string",
+            "example": "assistant"
+          },
+          "tool_calls": {
+            "$ref": "#/components/schemas/DeltaToolCall"
+          }
+        }
+      },
+      "ToolCallMessage": {
+        "type": "object",
+        "required": [
+          "role",
+          "tool_calls"
+        ],
+        "properties": {
+          "role": {
+            "type": "string",
+            "example": "assistant"
+          },
+          "tool_calls": {
+            "type": "array",
+            "items": {
+              "$ref": "#/components/schemas/ToolCall"
+            }
+          }
+        }
+      },
+      "ToolChoice": {
+        "allOf": [
+          {
+            "$ref": "#/components/schemas/ToolType"
+          }
+        ],
+        "nullable": true
+      },
       "ToolType": {
         "oneOf": [
+          {
+            "type": "object",
+            "default": null,
+            "nullable": true
+          },
+          {
+            "type": "string"
+          },
           {
             "type": "object",
             "required": [
-              "FunctionName"
+              "function"
             ],
             "properties": {
-              "FunctionName": {
-                "type": "string"
+              "function": {
+                "$ref": "#/components/schemas/FunctionName"
               }
             }
           },
           {
-            "type": "string",
-            "enum": [
-              "OneOf"
-            ]
+            "type": "object",
+            "default": null,
+            "nullable": true
           }
         ]
       },
+      "Url": {
+        "type": "object",
+        "required": [
+          "url"
+        ],
+        "properties": {
+          "url": {
+            "type": "string"
+          }
+        }
+      },
       "Usage": {
         "type": "object",
         "required": [
diff --git a/docs/source/_toctree.yml b/docs/source/_toctree.yml
index a7351a33..b883b36d 100644
--- a/docs/source/_toctree.yml
+++ b/docs/source/_toctree.yml
@@ -11,12 +11,16 @@
     title: Using TGI with Intel Gaudi
   - local: installation_inferentia
     title: Using TGI with AWS Inferentia
+  - local: installation_intel
+    title: Using TGI with Intel GPUs
   - local: installation
     title: Installation from source
   - local: supported_models
     title: Supported Models and Hardware
-  - local: messages_api
-    title: Messages API
+  - local: architecture
+    title: Internal Architecture
+  - local: usage_statistics
+    title: Usage Statistics
   title: Getting started
 - sections:
   - local: basic_tutorials/consuming_tgi
@@ -27,8 +31,6 @@
     title: Serving Private & Gated Models
   - local: basic_tutorials/using_cli
     title: Using TGI CLI
-  - local: basic_tutorials/launcher
-    title: All TGI CLI options
   - local: basic_tutorials/non_core_models
     title: Non-core Model Serving
   - local: basic_tutorials/safety
@@ -42,6 +44,14 @@
   - local: basic_tutorials/train_medusa
     title: Train Medusa
   title: Tutorials
+- sections:
+  - local: reference/launcher
+    title: All TGI CLI options
+  - local: reference/metrics
+    title: Exported Metrics
+  - local: reference/api_reference
+    title: API Reference
+  title: Reference
 - sections:
   - local: conceptual/streaming
     title: Streaming
@@ -59,5 +69,10 @@
     title: Speculation (Medusa, ngram)
   - local: conceptual/guidance
     title: How Guidance Works (via outlines)
+  - local: conceptual/lora
+    title: LoRA (Low-Rank Adaptation)
+  - local: conceptual/external
+    title: External Resources
+
 
   title: Conceptual Guides
diff --git a/docs/source/architecture.md b/docs/source/architecture.md
new file mode 100644
index 00000000..6660630d
--- /dev/null
+++ b/docs/source/architecture.md
@@ -0,0 +1,232 @@
+# Text Generation Inference Architecture
+
+This document aims at describing the architecture of Text Generation Inference (TGI), by describing the call flow between the separate components.
+
+A high-level architecture diagram can be seen here:
+
+![TGI architecture](https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/TGI.png)
+
+This diagram shows well there are these separate components:
+
+- **The router**, also named `webserver`, that receives the client requests, buffers them, creates some batches, and prepares gRPC calls to a model server.
+- **The model server**, responsible of receiving the gRPC requests and to process the inference on the model. If the model is sharded across multiple accelerators (e.g.: multiple GPUs), the model server shards might be synchronized via NCCL or equivalent.
+- **The launcher** is a helper that will be able to launch one or several model servers (if model is sharded), and it launches the router with the compatible arguments.
+
+The router and the model server can be two different machines, they do not need to be deployed together.
+
+## The Router
+
+This component is a rust web server binary that accepts HTTP requests using the custom [HTTP API](https://huggingface.github.io/text-generation-inference/), as well as OpenAI's [Messages API](https://huggingface.co/docs/text-generation-inference/messages_api).
+The router receives the API calls and handles the "baches" logic (and introduction to batching can be found [here](https://github.com/huggingface/text-generation-inference/blob/main/router/README.md)).
+It uses different strategies to reduce latency between requests and responses, especially oriented to decoding latency. It will use queues, schedulers, and block allocators to achieve that and produce batched requests that it will then be sent to the model server.
+
+### Router's command line
+
+The router command line will be the way to pass parameters to it (it does not rely on configuration file):
+
+```
+Text Generation Webserver
+
+Usage: text-generation-router [OPTIONS]
+
+Options:
+      --max-concurrent-requests <MAX_CONCURRENT_REQUESTS>
+          [env: MAX_CONCURRENT_REQUESTS=] [default: 128]
+      --max-best-of <MAX_BEST_OF>
+          [env: MAX_BEST_OF=] [default: 2]
+      --max-stop-sequences <MAX_STOP_SEQUENCES>
+          [env: MAX_STOP_SEQUENCES=] [default: 4]
+      --max-top-n-tokens <MAX_TOP_N_TOKENS>
+          [env: MAX_TOP_N_TOKENS=] [default: 5]
+      --max-input-tokens <MAX_INPUT_TOKENS>
+          [env: MAX_INPUT_TOKENS=] [default: 1024]
+      --max-total-tokens <MAX_TOTAL_TOKENS>
+          [env: MAX_TOTAL_TOKENS=] [default: 2048]
+      --waiting-served-ratio <WAITING_SERVED_RATIO>
+          [env: WAITING_SERVED_RATIO=] [default: 1.2]
+      --max-batch-prefill-tokens <MAX_BATCH_PREFILL_TOKENS>
+          [env: MAX_BATCH_PREFILL_TOKENS=] [default: 4096]
+      --max-batch-total-tokens <MAX_BATCH_TOTAL_TOKENS>
+          [env: MAX_BATCH_TOTAL_TOKENS=]
+      --max-waiting-tokens <MAX_WAITING_TOKENS>
+          [env: MAX_WAITING_TOKENS=] [default: 20]
+      --max-batch-size <MAX_BATCH_SIZE>
+          [env: MAX_BATCH_SIZE=]
+      --hostname <HOSTNAME>
+          [env: HOSTNAME=] [default: 0.0.0.0]
+  -p, --port <PORT>
+          [env: PORT=] [default: 3000]
+      --master-shard-uds-path <MASTER_SHARD_UDS_PATH>
+          [env: MASTER_SHARD_UDS_PATH=] [default: /tmp/text-generation-server-0]
+      --tokenizer-name <TOKENIZER_NAME>
+          [env: TOKENIZER_NAME=] [default: bigscience/bloom]
+      --tokenizer-config-path <TOKENIZER_CONFIG_PATH>
+          [env: TOKENIZER_CONFIG_PATH=]
+      --revision <REVISION>
+          [env: REVISION=]
+      --validation-workers <VALIDATION_WORKERS>
+          [env: VALIDATION_WORKERS=] [default: 2]
+      --json-output
+          [env: JSON_OUTPUT=]
+      --otlp-endpoint <OTLP_ENDPOINT>
+          [env: OTLP_ENDPOINT=]
+      --otlp-service-name <OTLP_SERVICE_NAME>
+          [env: OTLP_SERVICE_NAME=]
+      --cors-allow-origin <CORS_ALLOW_ORIGIN>
+          [env: CORS_ALLOW_ORIGIN=]
+      --ngrok
+          [env: NGROK=]
+      --ngrok-authtoken <NGROK_AUTHTOKEN>
+          [env: NGROK_AUTHTOKEN=]
+      --ngrok-edge <NGROK_EDGE>
+          [env: NGROK_EDGE=]
+      --messages-api-enabled
+          [env: MESSAGES_API_ENABLED=]
+      --disable-grammar-support
+          [env: DISABLE_GRAMMAR_SUPPORT=]
+      --max-client-batch-size <MAX_CLIENT_BATCH_SIZE>
+          [env: MAX_CLIENT_BATCH_SIZE=] [default: 4]
+  -h, --help
+          Print help
+  -V, --version
+          Print version
+```
+
+## The Model Server
+
+The model server is a python server, capable of starting a server waiting for gRPC requests, loads a given model, perform sharding to provide [tensor parallelism](https://huggingface.co/docs/text-generation-inference/conceptual/tensor_parallelism), and stays alive while waiting for new requests.
+The model server supports models instantiated using Pytorch and optimized for inference mainly on CUDA/ROCM.
+
+### Model Server Variants
+
+Several variants of the model server exist that are actively supported by Hugging Face:
+
+- By default, the model server will attempt building [a server optimized for Nvidia GPUs with CUDA](https://huggingface.co/docs/text-generation-inference/installation_nvidia). The code for this version is hosted in the [main TGI repository](https://github.com/huggingface/text-generation-inference).
+- A [version optimized for AMD with ROCm](https://huggingface.co/docs/text-generation-inference/installation_amd) is hosted in the main TGI repository. Some model features differ.
+- A [version optimized for Intel GPUs](https://huggingface.co/docs/text-generation-inference/installation_intel) is hosted in the main TGI repository. Some model features differ.
+- The [version for Intel Gaudi](https://huggingface.co/docs/text-generation-inference/installation_gaudi) is maintained on a forked repository, often resynchronized with the main [TGI repository](https://github.com/huggingface/tgi-gaudi).
+- A [version for Neuron (AWS Inferentia2)](https://huggingface.co/docs/text-generation-inference/installation_inferentia) is maintained as part of [Optimum Neuron](https://github.com/huggingface/optimum-neuron/tree/main/text-generation-inference).
+- A version for Google TPUs is maintained as part of [Optimum TPU](https://github.com/huggingface/optimum-tpu/tree/main/text-generation-inference).
+
+Not all variants provide the same features, as hardware and middleware capabilities do not provide the same optimizations.
+
+### Command Line Interface
+
+The official command line interface (CLI) for the server supports three subcommands, `download-weights`, `quantize` and `serve`:
+
+- `download-weights` will download weights from the hub and, in some variants it will convert weights to a format that is adapted to the given implementation;
+- `quantize` will allow to quantize a model using the `qptq` package. This feature is not available nor supported on all variants;
+- `serve` will start the server that load a model (or a model shard), receives gRPC calls from the router, performs an inference and provides a formatted response to the given request.
+
+Serve's command line parameters on the TGI repository are these:
+
+```
+ Usage: cli.py serve [OPTIONS] MODEL_ID
+
+╭─ Arguments ──────────────────────────────────────────────────────────────────────────────────────────────╮
+│ *    model_id      TEXT  [default: None] [required]                                                      │
+╰──────────────────────────────────────────────────────────────────────────────────────────────────────────╯
+╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────╮
+│ --revision                                       TEXT                        [default: None]             │
+│ --sharded              --no-sharded                                          [default: no-sharded]       │
+│ --quantize                                       [bitsandbytes|bitsandbytes  [default: None]             │
+│                                                  -nf4|bitsandbytes-fp4|gptq                              │
+│                                                  |awq|eetq|exl2|fp8]                                     │
+│ --speculate                                      INTEGER                     [default: None]             │
+│ --dtype                                          [float16|bfloat16]          [default: None]             │
+│ --trust-remote-code    --no-trust-remote-code                                [default:                   │
+│                                                                              no-trust-remote-code]       │
+│ --uds-path                                       PATH                        [default:                   │
+│                                                                              /tmp/text-generation-serve… │
+│ --logger-level                                   TEXT                        [default: INFO]             │
+│ --json-output          --no-json-output                                      [default: no-json-output]   │
+│ --otlp-endpoint                                  TEXT                        [default: None]             │
+│ --otlp-service-name                              TEXT                        [default:                   │
+│                                                                              text-generation-inference...│
+│ --help                                                                       Show this message and exit. │
+╰──────────────────────────────────────────────────────────────────────────────────────────────────────────╯
+```
+
+Note that some variants might support different parameters, and they could possibly accept more options that can be passed on using environment variables.
+
+## Call Flow
+
+Once both components are initialized, weights downloaded and model server is up and running, router and model server exchange data and info through the gRPC call. There are currently two supported schemas, [v2](https://github.com/huggingface/text-generation-inference/blob/main/proto/generate.proto) and [v3](https://github.com/huggingface/text-generation-inference/blob/main/proto/v3/generate.proto). These two versions are almost identical, except for:
+
+- input chunks support, for text and image data,
+- paged attention support
+
+Here's a diagram that displays the exchanges that follow the router and model server startup.
+
+```mermaid
+sequenceDiagram
+
+    Router->>Model Server: service discovery
+    Model Server-->>Router: urls for other shards
+
+    Router->>Model Server: get model info
+    Model Server-->>Router: shard info
+
+    Router->>Model Server: health check
+    Model Server-->>Router: health OK
+
+    Router->>Model Server: warmup(max_input_tokens, max_batch_prefill_tokens, max_total_tokens, max_batch_size)
+    Model Server-->>Router: warmup result
+```
+
+After these are done, the router is ready to receive generate calls from multiple clients. Here's an example.
+
+```mermaid
+sequenceDiagram
+    participant Client 1
+    participant Client 2
+    participant Client 3
+    participant Router
+    participant Model Server
+
+    Client 1->>Router: generate_stream
+    Router->>Model Server: prefill(batch1)
+    Model Server-->>Router: generations, cached_batch1, timings
+    Router-->>Client 1: token 1
+
+    Router->>Model Server: decode(cached_batch1)
+    Model Server-->>Router: generations, cached_batch1, timings
+    Router-->>Client 1: token 2
+
+    Router->>Model Server: decode(cached_batch1)
+    Model Server-->>Router: generations, cached_batch1, timings
+    Router-->>Client 1: token 3
+
+    Client 2->>Router: generate_stream
+    Router->>Model Server: prefill(batch2)
+    Note right of Model Server: This stops previous batch, that is restarted
+    Model Server-->>Router: generations, cached_batch2, timings
+    Router-->>Client 2: token 1'
+
+    Router->>Model Server: decode(cached_batch1, cached_batch2)
+    Model Server-->>Router: generations, cached_batch1, timings
+    Router-->>Client 1: token 4
+    Router-->>Client 2: token 2'
+
+    Note left of Client 1: Client 1 leaves
+    Router->>Model Server: filter_batch(cached_batch1, request_ids_to_keep=batch2)
+    Model Server-->>Router: filtered batch
+
+    Router->>Model Server: decode(cached_batch2)
+    Model Server-->>Router: generations, cached_batch2, timings
+    Router-->>Client 2: token 3'
+
+    Client 3->>Router: generate_stream
+    Note right of Model Server: This stops previous batch, that is restarted
+    Router->>Model Server: prefill(batch3)
+    Note left of Client 1: Client 3 leaves without receiving any batch
+    Router->>Model Server: clear_cache(batch3)
+    Note right of Model Server: This stops previous batch, that is restarted
+
+    Router->>Model Server: decode(cached_batch3)
+    Note right of Model Server: Last token (stopping criteria)
+    Model Server-->>Router: generations, cached_batch3, timings
+    Router-->>Client 2: token 4'
+
+
+```
diff --git a/docs/source/basic_tutorials/consuming_tgi.md b/docs/source/basic_tutorials/consuming_tgi.md
index 4829ec7c..b07e7219 100644
--- a/docs/source/basic_tutorials/consuming_tgi.md
+++ b/docs/source/basic_tutorials/consuming_tgi.md
@@ -1,81 +1,125 @@
 # Consuming Text Generation Inference
 
-There are many ways you can consume Text Generation Inference server in your applications. After launching, you can use the `/generate` route and make a `POST` request to get results from the server. You can also use the `/generate_stream` route if you want TGI to return a stream of tokens. You can make the requests using the tool of your preference, such as curl, Python or TypeScrpt. For a final end-to-end experience, we also open-sourced ChatUI, a chat interface for open-source models.
+There are many ways to consume Text Generation Inference (TGI) server in your applications. After launching the server, you can use the [Messages API](https://huggingface.co/docs/text-generation-inference/en/messages_api) `/v1/chat/completions` route and make a `POST` request to get results from the server. You can also pass `"stream": true` to the call if you want TGI to return a stream of tokens.
+
+For more information on the API, consult the OpenAPI documentation of `text-generation-inference` available [here](https://huggingface.github.io/text-generation-inference).
+
+You can make the requests using any tool of your preference, such as curl, Python, or TypeScript. For an end-to-end experience, we've open-sourced [ChatUI](https://github.com/huggingface/chat-ui), a chat interface for open-access models.
 
 ## curl
 
-After the launch, you can query the model using either the `/generate` or `/generate_stream` routes:
+After a successful server launch, you can query the model using the `v1/chat/completions` route, to get responses that are compliant to the OpenAI Chat Completion spec:
+
+```bash
+curl localhost:8080/v1/chat/completions \
+    -X POST \
+    -d '{
+  "model": "tgi",
+  "messages": [
+    {
+      "role": "system",
+      "content": "You are a helpful assistant."
+    },
+    {
+      "role": "user",
+      "content": "What is deep learning?"
+    }
+  ],
+  "stream": true,
+  "max_tokens": 20
+}' \
+    -H 'Content-Type: application/json'
+```
+
+For non-chat use-cases, you can also use the `/generate` and `/generate_stream` routes.
 
 ```bash
 curl 127.0.0.1:8080/generate \
     -X POST \
-    -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":20}}' \
+    -d '{
+  "inputs":"What is Deep Learning?",
+  "parameters":{
+    "max_new_tokens":20
+  }
+}' \
     -H 'Content-Type: application/json'
 ```
 
+## Python
 
-## Inference Client
+### Inference Client
 
-[`huggingface-hub`](https://huggingface.co/docs/huggingface_hub/main/en/index) is a Python library to interact with the Hugging Face Hub, including its endpoints. It provides a nice high-level class, [`~huggingface_hub.InferenceClient`], which makes it easy to make calls to a TGI endpoint. `InferenceClient` also takes care of parameter validation and provides a simple to-use interface.
-You can simply install `huggingface-hub` package with pip.
+[`huggingface_hub`](https://huggingface.co/docs/huggingface_hub/main/en/index) is a Python library to interact with the Hugging Face Hub, including its endpoints. It provides a high-level class, [`huggingface_hub.InferenceClient`](https://huggingface.co/docs/huggingface_hub/package_reference/inference_client#huggingface_hub.InferenceClient), which makes it easy to make calls to TGI's Messages API. `InferenceClient` also takes care of parameter validation and provides a simple-to-use interface.
+
+Install `huggingface_hub` package via pip.
 
 ```bash
-pip install huggingface-hub
+pip install huggingface_hub
 ```
 
-Once you start the TGI server, instantiate `InferenceClient()` with the URL to the endpoint serving the model. You can then call `text_generation()` to hit the endpoint through Python.
+You can now use `InferenceClient` the exact same way you would use `OpenAI` client in Python
 
 ```python
 from huggingface_hub import InferenceClient
 
-client = InferenceClient(model="http://127.0.0.1:8080")
-client.text_generation(prompt="Write a code for snake game")
+client = InferenceClient(
+    base_url="http://localhost:8080/v1/",
+)
+
+output = client.chat.completions.create(
+    model="tgi",
+    messages=[
+        {"role": "system", "content": "You are a helpful assistant."},
+        {"role": "user", "content": "Count to 10"},
+    ],
+    stream=True,
+    max_tokens=1024,
+)
+
+for chunk in output:
+    print(chunk.choices[0].delta.content)
 ```
 
-You can do streaming with `InferenceClient` by passing `stream=True`. Streaming will return tokens as they are being generated in the server. To use streaming, you can do as follows:
+You can check out more details about OpenAI compatibility [here](https://huggingface.co/docs/huggingface_hub/en/guides/inference#openai-compatibility).
+
+There is also an async version of the client, `AsyncInferenceClient`, based on `asyncio` and `aiohttp`. You can find docs for it [here](https://huggingface.co/docs/huggingface_hub/package_reference/inference_client#huggingface_hub.AsyncInferenceClient)
+
+### OpenAI Client
+
+You can directly use the OpenAI [Python](https://github.com/openai/openai-python) or [JS](https://github.com/openai/openai-node) clients to interact with TGI.
+
+Install the OpenAI Python package via pip.
+
+```bash
+pip install openai
+```
 
 ```python
-for token in client.text_generation("How do you make cheese?", max_new_tokens=12, stream=True):
-    print(token)
+from openai import OpenAI
+
+# init the client but point it to TGI
+client = OpenAI(
+    base_url="http://localhost:8080/v1/",
+    api_key="-"
+)
+
+chat_completion = client.chat.completions.create(
+    model="tgi",
+    messages=[
+        {"role": "system", "content": "You are a helpful assistant." },
+        {"role": "user", "content": "What is deep learning?"}
+    ],
+    stream=True
+)
+
+# iterate and print stream
+for message in chat_completion:
+    print(message)
 ```
 
-Another parameter you can use with TGI backend is `details`. You can get more details on generation (tokens, probabilities, etc.) by setting `details` to `True`. When it's specified, TGI will return a `TextGenerationResponse` or `TextGenerationStreamResponse` rather than a string or stream.
+## UI
 
-```python
-output = client.text_generation(prompt="Meaning of life is", details=True)
-print(output)
-
-# TextGenerationResponse(generated_text=' a complex concept that is not always clear to the individual. It is a concept that is not always', details=Details(finish_reason=<FinishReason.Length: 'length'>, generated_tokens=20, seed=None, prefill=[], tokens=[Token(id=267, text=' a', logprob=-2.0723474, special=False), Token(id=11235, text=' complex', logprob=-3.1272552, special=False), Token(id=17908, text=' concept', logprob=-1.3632495, special=False),..))
-```
-
-You can see how to stream below.
-
-```python
-output = client.text_generation(prompt="Meaning of life is", stream=True, details=True)
-print(next(iter(output)))
-
-# TextGenerationStreamResponse(token=Token(id=267, text=' a', logprob=-2.0723474, special=False), generated_text=None, details=None)
-```
-
-You can check out the details of the function [here](https://huggingface.co/docs/huggingface_hub/main/en/package_reference/inference_client#huggingface_hub.InferenceClient.text_generation). There is also an async version of the client, `AsyncInferenceClient`, based on `asyncio` and `aiohttp`. You can find docs for it [here](https://huggingface.co/docs/huggingface_hub/package_reference/inference_client#huggingface_hub.AsyncInferenceClient)
-
-
-## ChatUI
-
-ChatUI is an open-source interface built for LLM serving. It offers many customization options, such as web search with SERP API and more. ChatUI can automatically consume the TGI server and even provides an option to switch between different TGI endpoints. You can try it out at [Hugging Chat](https://huggingface.co/chat/), or use the [ChatUI Docker Space](https://huggingface.co/new-space?template=huggingchat/chat-ui-template) to deploy your own Hugging Chat to Spaces.
-
-To serve both ChatUI and TGI in same environment, simply add your own endpoints to the `MODELS` variable in `.env.local` file inside the `chat-ui` repository. Provide the endpoints pointing to where TGI is served.
-
-```
-{
-// rest of the model config here
-"endpoints": [{"url": "https://HOST:PORT/generate_stream"}]
-}
-```
-
-![ChatUI](https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/chatui_screen.png)
-
-## Gradio
+### Gradio
 
 Gradio is a Python library that helps you build web applications for your machine learning models with a few lines of code. It has a `ChatInterface` wrapper that helps create neat UIs for chatbots. Let's take a look at how to create a chatbot with streaming mode using TGI and Gradio. Let's install Gradio and Hub Python library first.
 
@@ -89,19 +133,28 @@ Assume you are serving your model on port 8080, we will query through [Inference
 import gradio as gr
 from huggingface_hub import InferenceClient
 
-client = InferenceClient(model="http://127.0.0.1:8080")
+client = InferenceClient(base_url="http://127.0.0.1:8080")
 
 def inference(message, history):
     partial_message = ""
-    for token in client.text_generation(message, max_new_tokens=20, stream=True):
-        partial_message += token
+    output = client.chat.completions.create(
+        messages=[
+            {"role": "system", "content": "You are a helpful assistant."},
+            {"role": "user", "content": message},
+        ],
+        stream=True,
+        max_tokens=1024,
+    )
+
+    for chunk in output:
+        partial_message += chunk.choices[0].delta.content
         yield partial_message
 
 gr.ChatInterface(
     inference,
     chatbot=gr.Chatbot(height=300),
     textbox=gr.Textbox(placeholder="Chat with me!", container=False, scale=7),
-    description="This is the demo for Gradio UI consuming TGI endpoint with LLaMA 7B-Chat model.",
+    description="This is the demo for Gradio UI consuming TGI endpoint.",
     title="Gradio 🤝 TGI",
     examples=["Are tomatoes vegetables?"],
     retry_btn="Retry",
@@ -110,20 +163,7 @@ gr.ChatInterface(
 ).queue().launch()
 ```
 
-The UI looks like this 👇
-
-<div class="flex justify-center">
-    <img
-        class="block dark:hidden"
-        src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/tgi/gradio-tgi.png"
-    />
-    <img
-        class="hidden dark:block"
-        src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/tgi/gradio-tgi-dark.png"
-    />
-</div>
-
-You can try the demo directly here 👇
+You can check out the UI and try the demo directly here 👇
 
 <div class="block dark:hidden">
 	<iframe
@@ -141,15 +181,19 @@ You can try the demo directly here 👇
 </div>
 
 
-You can disable streaming mode using `return` instead of `yield` in your inference function, like below.
-
-```python
-def inference(message, history):
-    return client.text_generation(message, max_new_tokens=20)
-```
-
 You can read more about how to customize a `ChatInterface` [here](https://www.gradio.app/guides/creating-a-chatbot-fast).
 
-## API documentation
+### ChatUI
 
-You can consult the OpenAPI documentation of the `text-generation-inference` REST API using the `/docs` route. The Swagger UI is also available [here](https://huggingface.github.io/text-generation-inference).
+[ChatUI](https://github.com/huggingface/chat-ui) is an open-source interface built for consuming LLMs. It offers many customization options, such as web search with SERP API and more. ChatUI can automatically consume the TGI server and even provides an option to switch between different TGI endpoints. You can try it out at [Hugging Chat](https://huggingface.co/chat/), or use the [ChatUI Docker Space](https://huggingface.co/new-space?template=huggingchat/chat-ui-template) to deploy your own Hugging Chat to Spaces.
+
+To serve both ChatUI and TGI in same environment, simply add your own endpoints to the `MODELS` variable in `.env.local` file inside the `chat-ui` repository. Provide the endpoints pointing to where TGI is served.
+
+```
+{
+// rest of the model config here
+"endpoints": [{"url": "https://HOST:PORT/generate_stream"}]
+}
+```
+
+![ChatUI](https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/chatui_screen.png)
diff --git a/docs/source/basic_tutorials/gated_model_access.md b/docs/source/basic_tutorials/gated_model_access.md
index 2999e2e0..ef3a1db7 100644
--- a/docs/source/basic_tutorials/gated_model_access.md
+++ b/docs/source/basic_tutorials/gated_model_access.md
@@ -19,6 +19,6 @@ docker run --gpus all \
     --shm-size 1g \
     -e HF_TOKEN=$token \
     -p 8080:80 \
-    -v $volume:/data ghcr.io/huggingface/text-generation-inference:2.0.3 \
+    -v $volume:/data ghcr.io/huggingface/text-generation-inference:2.0.4 \
     --model-id $model
 ```
diff --git a/docs/source/basic_tutorials/preparing_model.md b/docs/source/basic_tutorials/preparing_model.md
index 71ca5598..456ade44 100644
--- a/docs/source/basic_tutorials/preparing_model.md
+++ b/docs/source/basic_tutorials/preparing_model.md
@@ -4,7 +4,7 @@ Text Generation Inference improves the model in several aspects.
 
 ## Quantization
 
-TGI supports [bits-and-bytes](https://github.com/TimDettmers/bitsandbytes#bitsandbytes), [GPT-Q](https://arxiv.org/abs/2210.17323) and [AWQ](https://arxiv.org/abs/2306.00978) quantization. To speed up inference with quantization, simply set `quantize` flag to `bitsandbytes`, `gptq` or `awq` depending on the quantization technique you wish to use. When using GPT-Q quantization, you need to point to one of the models [here](https://huggingface.co/models?search=gptq) when using AWQ quantization, you need to point to one of the models [here](https://huggingface.co/models?search=awq). To get more information about quantization, please refer to [quantization guide](./../conceptual/quantization)
+TGI supports [bits-and-bytes](https://github.com/TimDettmers/bitsandbytes#bitsandbytes), [GPT-Q](https://arxiv.org/abs/2210.17323), [AWQ](https://arxiv.org/abs/2306.00978), [Marlin](https://github.com/IST-DASLab/marlin), [EETQ](https://github.com/NetEase-FuXi/EETQ), [EXL2](https://github.com/turboderp/exllamav2), and [fp8](https://developer.nvidia.com/blog/nvidia-arm-and-intel-publish-fp8-specification-for-standardization-as-an-interchange-format-for-ai/) quantization. To speed up inference with quantization, simply set `quantize` flag to `bitsandbytes`, `gptq`, `awq`, `marlin`, `exl2`, `eetq` or `fp8` depending on the quantization technique you wish to use. When using GPT-Q quantization, you need to point to one of the models [here](https://huggingface.co/models?search=gptq). Similarly, when using AWQ quantization, you need to point to one of [these models](https://huggingface.co/models?search=awq). To get more information about quantization, please refer to [quantization guide](./../conceptual/quantization)
 
 
 ## RoPE Scaling
diff --git a/docs/source/basic_tutorials/train_medusa.md b/docs/source/basic_tutorials/train_medusa.md
index 76cb6bed..ba2e43b7 100644
--- a/docs/source/basic_tutorials/train_medusa.md
+++ b/docs/source/basic_tutorials/train_medusa.md
@@ -1,6 +1,6 @@
 # Train Medusa
 
-This tutorial will show you how to train a Medusa model on a dataset of your choice. Please check out the [speculation documentation](../conceptual/speculation.md) for more information on how Medusa works and speculation in general.
+This tutorial will show you how to train a Medusa model on a dataset of your choice. Please check out the [speculation documentation](../conceptual/speculation) for more information on how Medusa works and speculation in general.
 
 ## What are the benefits of training a Medusa model?
 
diff --git a/docs/source/basic_tutorials/using_guidance.md b/docs/source/basic_tutorials/using_guidance.md
index d0008fdb..7e33e9a2 100644
--- a/docs/source/basic_tutorials/using_guidance.md
+++ b/docs/source/basic_tutorials/using_guidance.md
@@ -4,7 +4,7 @@ Text Generation Inference (TGI) now supports [JSON and regex grammars](#grammar-
 
 These feature are available starting from version `1.4.3`. They are accessible via the [`huggingface_hub`](https://pypi.org/project/huggingface-hub/) library. The tool support is compatible with OpenAI's client libraries. The following guide will walk you through the new features and how to use them!
 
-_note: guidance is supported as grammar in the `/generate` endpoint and as tools in the `/chat/completions` endpoint._
+_note: guidance is supported as grammar in the `/generate` endpoint and as tools in the `v1/chat/completions` endpoint._
 
 ## How it works
 
@@ -157,7 +157,12 @@ from huggingface_hub import InferenceClient
 
 client = InferenceClient("http://localhost:3000")
 
-regexp = "((25[0-5]|2[0-4]\\d|[01]?\\d\\d?)\\.){3}(25[0-5]|2[0-4]\\d|[01]?\\d\\d?)"
+section_regex = "(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)"
+regexp = f"HELLO\.{section_regex}\.WORLD\.{section_regex}"
+
+# This is a more realistic example of an ip address regex
+# regexp = f"{section_regex}\.{section_regex}\.{section_regex}\.{section_regex}"
+
 
 resp = client.text_generation(
     f"Whats Googles DNS? Please use the following regex: {regexp}",
@@ -170,7 +175,7 @@ resp = client.text_generation(
 
 
 print(resp)
-# 7.1.1.1
+# HELLO.255.WORLD.255
 
 ```
 
@@ -306,11 +311,13 @@ print(chat.choices[0].message.tool_calls)
 
 ```
 
-### OpenAI integration
+### OpenAI Integration
 
-TGI exposes an OpenAI-compatible API, which means you can use OpenAI's client libraries to interact with TGI's Messages API and Tool functions.
+Text Generation Inference (TGI) offers seamless integration with OpenAI's client libraries, allowing developers to interact with TGI's Messages API and Tool functions in a familiar way. This compatibility simplifies the implementation of advanced features, such as tools and grammar, within your applications using OpenAI’s client.
 
-However there are some minor differences in the API, for example `tool_choice="auto"` will ALWAYS choose the tool for you. This is different from OpenAI's API where `tool_choice="auto"` will choose a tool if the model thinks it's necessary.
+Previously, TGI handled tool selection differently than OpenAI’s API—`tool_choice="auto"` would always pick a tool for you. However, as of the latest version, TGI now mimics OpenAI’s behavior more closely: `tool_choice="auto"` selects a tool only when the model deems it necessary, aligning with how OpenAI's API works. This enhancement ensures a smoother and more predictable integration experience.
+
+Additionally, error notifications like `notify_error`, which previously indicated that no tool was chosen, are no longer returned. Instead, TGI will proceed with generating a response as if no tool was selected, further improving consistency with OpenAI's API.
 
 ```python
 from openai import OpenAI
diff --git a/docs/source/basic_tutorials/visual_language_models.md b/docs/source/basic_tutorials/visual_language_models.md
index 3770db0b..f152a2f0 100644
--- a/docs/source/basic_tutorials/visual_language_models.md
+++ b/docs/source/basic_tutorials/visual_language_models.md
@@ -84,7 +84,7 @@ print(chat)
 
 ```
 
-or with OpenAi's library:
+or with OpenAI's [client library](https://github.com/openai/openai-python):
 
 ```python
 from openai import OpenAI
diff --git a/docs/source/conceptual/external.md b/docs/source/conceptual/external.md
new file mode 100644
index 00000000..9cbe1b5a
--- /dev/null
+++ b/docs/source/conceptual/external.md
@@ -0,0 +1,4 @@
+# External Resources
+
+- Adyen wrote a detailed article about the interplay between TGI's main components: router and server.
+[LLM inference at scale with TGI (Martin Iglesias Goyanes - Adyen, 2024)](https://www.adyen.com/knowledge-hub/llm-inference-at-scale-with-tgi)
diff --git a/docs/source/conceptual/guidance.md b/docs/source/conceptual/guidance.md
index a566c4a6..3059e3de 100644
--- a/docs/source/conceptual/guidance.md
+++ b/docs/source/conceptual/guidance.md
@@ -2,11 +2,11 @@
 
 ## What is Guidance?
 
-Guidance is a feature that allows users to constrain the generation of a large language model with a specified grammar. This feature is particularly useful when you want to generate text that follows a specific structure or uses a specific set of words or produce output in a specific format.
+Guidance is a feature that allows users to constrain the generation of a large language model with a specified grammar. This feature is particularly useful when you want to generate text that follows a specific structure or uses a specific set of words or produce output in a specific format. A prominent example is JSON grammar, where the model is forced to output valid JSON.
 
 ## How is it used?
 
-Guidance can be in many ways and the community is always finding new ways to use it. Here are some examples of how you can use guidance:
+Guidance can be implemented in many ways and the community is always finding new ways to use it. Here are some examples of how you can use guidance:
 
 Technically, guidance can be used to generate:
 
diff --git a/docs/source/conceptual/lora.md b/docs/source/conceptual/lora.md
new file mode 100644
index 00000000..d1f4ce78
--- /dev/null
+++ b/docs/source/conceptual/lora.md
@@ -0,0 +1,99 @@
+# LoRA (Low-Rank Adaptation)
+
+## What is LoRA?
+
+LoRA is a technique that allows for efficent fine-tuning a model while only updating a small portion of the model's weights. This is useful when you have a large model that has been pre-trained on a large dataset, but you want to fine-tune it on a smaller dataset or for a specific task.
+
+LoRA works by adding a small number of additional weights to the model, which are used to adapt the model to the new dataset or task. These additional weights are learned during the fine-tuning process, while the rest of the model's weights are kept fixed.
+
+## How is it used?
+
+LoRA can be used in many ways and the community is always finding new ways to use it. Here are some examples of how you can use LoRA:
+
+Technically, LoRA can be used to fine-tune a large language model on a small dataset. However, these use cases can span a wide range of applications, such as:
+
+- fine-tuning a language model on a small dataset
+- fine-tuning a language model on a domain-specific dataset
+- fine-tuning a language model on a dataset with limited labels
+
+## Optimizing Inference with LoRA
+
+LoRA's can be used during inference by mutliplying the adapter weights with the model weights at each specified layer. This process can be computationally expensive, but due to awesome work by [punica-ai](https://github.com/punica-ai/punica) and the [lorax](https://github.com/predibase/lorax) team, optimized kernels/and frameworks have been developed to make this process more efficient. TGI leverages these optimizations in order to provide fast and efficient inference with mulitple LoRA models.
+
+## Serving multiple LoRA adapters with TGI
+
+Once a LoRA model has been trained, it can be used to generate text or perform other tasks just like a regular language model. However, because the model has been fine-tuned on a specific dataset, it may perform better on that dataset than a model that has not been fine-tuned.
+
+In practice its often useful to have multiple LoRA models, each fine-tuned on a different dataset or for a different task. This allows you to use the model that is best suited for a particular task or dataset.
+
+Text Generation Inference (TGI) now supports loading multiple LoRA models at startup that can be used in generation requests. This feature is available starting from version `~2.0.6` and is compatible with LoRA models trained using the `peft` library.
+
+### Specifying LoRA models
+
+To use LoRA in TGI, when starting the server, you can specify the list of LoRA models to load using the `LORA_ADAPTERS` environment variable. For example:
+
+```bash
+LORA_ADAPTERS=predibase/customer_support,predibase/dbpedia
+```
+
+To specify model revision, use `adapter_id@revision`, as follows:
+
+```bash
+LORA_ADAPTERS=predibase/customer_support@main,predibase/dbpedia@rev2
+```
+
+To use a locally stored lora adapter, use `adapter-name=/path/to/adapter`, as seen below. When you want to use this adapter, set `"parameters": {"adapter_id": "adapter-name"}"`
+
+```bash
+LORA_ADAPTERS=myadapter=/some/path/to/adapter,myadapter2=/another/path/to/adapter
+```
+
+note it's possible to mix adapter_ids with adapter_id=adapter_path e.g.
+
+```bash
+LORA_ADAPTERS=predibase/dbpedia,myadapter=/path/to/dir/
+```
+
+In the server logs, you will see the following message:
+
+```txt
+Loading adapter weights into model: predibase/customer_support
+Loading adapter weights into model: predibase/dbpedia
+```
+
+## Generate text
+
+You can then use these models in generation requests by specifying the `lora_model` parameter in the request payload. For example:
+
+```json
+curl 127.0.0.1:3000/generate \
+    -X POST \
+    -H 'Content-Type: application/json' \
+    -d '{
+  "inputs": "Hello who are you?",
+  "parameters": {
+    "max_new_tokens": 40,
+    "adapter_id": "predibase/customer_support"
+  }
+}'
+```
+
+If you are using a lora adapter stored locally that was set in the following manner: `LORA_ADAPTERS=myadapter=/some/path/to/adapter`, here is an example payload:
+
+```json
+curl 127.0.0.1:3000/generate \
+    -X POST \
+    -H 'Content-Type: application/json' \
+    -d '{
+  "inputs": "Hello who are you?",
+  "parameters": {
+    "max_new_tokens": 40,
+    "adapter_id": "myadapter"
+  }
+}'
+```
+
+
+> **Note:** The Lora feature is new and still being improved. If you encounter any issues or have any feedback, please let us know by opening an issue on the [GitHub repository](https://github.com/huggingface/text-generation-inference/issues/new/choose). Additionally documentation and an improved client library will be published soon.
+
+An updated tutorial with detailed examples will be published soon. Stay tuned!
diff --git a/docs/source/conceptual/quantization.md b/docs/source/conceptual/quantization.md
index 8f26fdba..b7672a9f 100644
--- a/docs/source/conceptual/quantization.md
+++ b/docs/source/conceptual/quantization.md
@@ -1,6 +1,40 @@
 # Quantization
 
-TGI offers GPTQ and bits-and-bytes quantization to quantize large language models.
+TGI offers many quantization schemes to run LLMs effectively and fast based on your use-case. TGI supports GPTQ, AWQ, bits-and-bytes, EETQ, Marlin, EXL2 and fp8 quantization.
+
+To leverage GPTQ, AWQ, Marlin and EXL2 quants, you must provide pre-quantized weights. Whereas for bits-and-bytes, EETQ and fp8, weights are quantized by TGI on the fly.
+
+We recommend using the official quantization scripts for creating your quants:
+1. [AWQ](https://github.com/casper-hansen/AutoAWQ/blob/main/examples/quantize.py)
+2. [GPTQ/ Marlin](https://github.com/AutoGPTQ/AutoGPTQ/blob/main/examples/quantization/basic_usage.py)
+3. [EXL2](https://github.com/turboderp/exllamav2/blob/master/doc/convert.md)
+
+For on-the-fly quantization you simply need to pass one of the supported quantization types and TGI takes care of the rest.
+
+## Quantization with bitsandbytes, EETQ & fp8
+
+bitsandbytes is a library used to apply 8-bit and 4-bit quantization to models. Unlike GPTQ quantization, bitsandbytes doesn't require a calibration dataset or any post-processing – weights are automatically quantized on load. However, inference with bitsandbytes is slower than GPTQ or FP16 precision.
+
+8-bit quantization enables multi-billion parameter scale models to fit in smaller hardware without degrading performance too much.
+In TGI, you can use 8-bit quantization by adding `--quantize bitsandbytes` like below 👇
+
+```bash
+docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:latest --model-id $model --quantize bitsandbytes
+```
+
+4-bit quantization is also possible with bitsandbytes. You can choose one of the following 4-bit data types: 4-bit float (`fp4`), or 4-bit `NormalFloat` (`nf4`). These data types were introduced in the context of parameter-efficient fine-tuning, but you can apply them for inference by automatically converting the model weights on load.
+
+In TGI, you can use 4-bit quantization by adding `--quantize bitsandbytes-nf4` or `--quantize bitsandbytes-fp4` like below 👇
+
+```bash
+docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:latest --model-id $model --quantize bitsandbytes-nf4
+```
+
+You can get more information about 8-bit quantization by reading this [blog post](https://huggingface.co/blog/hf-bitsandbytes-integration), and 4-bit quantization by reading [this blog post](https://huggingface.co/blog/4bit-transformers-bitsandbytes).
+
+Similarly you can use pass you can pass `--quantize eetq` or `--quantize fp8` for respective quantization schemes.
+
+In addition to this, TGI allows creating GPTQ quants directly by passing the model weights and a calibration dataset.
 
 ## Quantization with GPTQ
 
@@ -36,24 +70,3 @@ You can learn more about the quantization options by running `text-generation-se
 
 If you wish to do more with GPTQ models (e.g. train an adapter on top), you can read about transformers GPTQ integration [here](https://huggingface.co/blog/gptq-integration).
 You can learn more about GPTQ from the [paper](https://arxiv.org/pdf/2210.17323.pdf).
-
-## Quantization with bitsandbytes
-
-bitsandbytes is a library used to apply 8-bit and 4-bit quantization to models. Unlike GPTQ quantization, bitsandbytes doesn't require a calibration dataset or any post-processing – weights are automatically quantized on load. However, inference with bitsandbytes is slower than GPTQ or FP16 precision.
-
-8-bit quantization enables multi-billion parameter scale models to fit in smaller hardware without degrading performance too much.
-In TGI, you can use 8-bit quantization by adding `--quantize bitsandbytes` like below 👇
-
-```bash
-docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:latest --model-id $model --quantize bitsandbytes
-```
-
-4-bit quantization is also possible with bitsandbytes. You can choose one of the following 4-bit data types: 4-bit float (`fp4`), or 4-bit `NormalFloat` (`nf4`). These data types were introduced in the context of parameter-efficient fine-tuning, but you can apply them for inference by automatically converting the model weights on load.
-
-In TGI, you can use 4-bit quantization by adding `--quantize bitsandbytes-nf4` or `--quantize bitsandbytes-fp4` like below 👇
-
-```bash
-docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:latest --model-id $model --quantize bitsandbytes-nf4
-```
-
-You can get more information about 8-bit quantization by reading this [blog post](https://huggingface.co/blog/hf-bitsandbytes-integration), and 4-bit quantization by reading [this blog post](https://huggingface.co/blog/4bit-transformers-bitsandbytes).
diff --git a/docs/source/conceptual/streaming.md b/docs/source/conceptual/streaming.md
index 71ec9b25..b8154ba4 100644
--- a/docs/source/conceptual/streaming.md
+++ b/docs/source/conceptual/streaming.md
@@ -1,5 +1,6 @@
 # Streaming
 
+
 ## What is Streaming?
 
 Token streaming is the mode in which the server returns the tokens one by one as the model generates them. This enables showing progressive generations to the user rather than waiting for the whole generation. Streaming is an essential aspect of the end-user experience as it reduces latency, one of the most critical aspects of a smooth experience.
@@ -48,34 +49,29 @@ To stream tokens with `InferenceClient`, simply pass `stream=True` and iterate o
 ```python
 from huggingface_hub import InferenceClient
 
-client = InferenceClient("http://127.0.0.1:8080")
-for token in client.text_generation("How do you make cheese?", max_new_tokens=12, stream=True):
-    print(token)
+client = InferenceClient(base_url="http://127.0.0.1:8080")
+output = client.chat.completions.create(
+    messages=[
+        {"role": "system", "content": "You are a helpful assistant."},
+        {"role": "user", "content": "Count to 10"},
+    ],
+    stream=True,
+    max_tokens=1024,
+)
 
-# To
-# make
-# cheese
-#,
-# you
-# need
-# to
-# start
-# with
-# milk
-#.
-```
+for chunk in output:
+    print(chunk.choices[0].delta.content)
 
-If you want additional details, you can add `details=True`. In this case, you get a `TextGenerationStreamResponse` which contains additional information such as the probabilities and the tokens. For the final response in the stream, it also returns the full generated text.
-
-```python
-for details in client.text_generation("How do you make cheese?", max_new_tokens=12, details=True, stream=True):
-    print(details)
-
-#TextGenerationStreamResponse(token=Token(id=193, text='\n', logprob=-0.007358551, special=False), generated_text=None, details=None)
-#TextGenerationStreamResponse(token=Token(id=2044, text='To', logprob=-1.1357422, special=False), generated_text=None, details=None)
-#TextGenerationStreamResponse(token=Token(id=717, text=' make', logprob=-0.009841919, special=False), generated_text=None, details=None)
-#...
-#TextGenerationStreamResponse(token=Token(id=25, text='.', logprob=-1.3408203, special=False), generated_text='\nTo make cheese, you need to start with milk.', details=StreamDetails(finish_reason=<FinishReason.Length: 'length'>, generated_tokens=12, seed=None))
+# 1
+# 2
+# 3
+# 4
+# 5
+# 6
+# 7
+# 8
+# 9
+# 10
 ```
 
 The `huggingface_hub` library also comes with an `AsyncInferenceClient` in case you need to handle the requests concurrently.
@@ -83,31 +79,46 @@ The `huggingface_hub` library also comes with an `AsyncInferenceClient` in case
 ```python
 from huggingface_hub import AsyncInferenceClient
 
-client = AsyncInferenceClient("http://127.0.0.1:8080")
-async for token in await client.text_generation("How do you make cheese?", stream=True):
-    print(token)
+client = AsyncInferenceClient(base_url="http://127.0.0.1:8080")
+async def main():
+    stream = await client.chat.completions.create(
+        messages=[{"role": "user", "content": "Say this is a test"}],
+        stream=True,
+    )
+    async for chunk in stream:
+        print(chunk.choices[0].delta.content or "", end="")
 
-# To
-# make
-# cheese
-#,
-# you
-# need
-# to
-# start
-# with
-# milk
+asyncio.run(main())
+
+# This
+# is
+# a
+# test
 #.
 ```
 
 ### Streaming with cURL
 
-To use the `generate_stream` endpoint with curl, you can add the `-N` flag, which disables curl default buffering and shows data as it arrives from the server
+To use the OpenAI Chat Completions compatible Messages API `v1/chat/completions` endpoint with curl, you can add the `-N` flag, which disables curl default buffering and shows data as it arrives from the server
 
 ```curl
-curl -N 127.0.0.1:8080/generate_stream \
+curl localhost:8080/v1/chat/completions \
     -X POST \
-    -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":20}}' \
+    -d '{
+  "model": "tgi",
+  "messages": [
+    {
+      "role": "system",
+      "content": "You are a helpful assistant."
+    },
+    {
+      "role": "user",
+      "content": "What is deep learning?"
+    }
+  ],
+  "stream": true,
+  "max_tokens": 20
+}' \
     -H 'Content-Type: application/json'
 ```
 
diff --git a/docs/source/installation_amd.md b/docs/source/installation_amd.md
index 636d301c..86d092eb 100644
--- a/docs/source/installation_amd.md
+++ b/docs/source/installation_amd.md
@@ -11,7 +11,7 @@ volume=$PWD/data # share a volume with the Docker container to avoid downloading
 docker run --rm -it --cap-add=SYS_PTRACE --security-opt seccomp=unconfined \
     --device=/dev/kfd --device=/dev/dri --group-add video \
     --ipc=host --shm-size 256g --net host -v $volume:/data \
-    ghcr.io/huggingface/text-generation-inference:2.0.3-rocm \
+    ghcr.io/huggingface/text-generation-inference:2.3.1-rocm \
     --model-id $model
 ```
 
@@ -27,10 +27,16 @@ TunableOp is enabled by default, the warmup may take 1-2 minutes. In case you wo
 
 ## Flash attention implementation
 
-Two implementations of Flash Attention are available for ROCm, the first is [ROCm/flash-attention](https://github.com/ROCm/flash-attention) based on a [Composable Kernel](https://github.com/ROCm/composable_kernel) (CK) implementation, and the second is a [Triton implementation](https://github.com/huggingface/text-generation-inference/blob/main/server/text_generation_server/utils/flash_attn_triton.py).
+Two implementations of Flash Attention are available for ROCm, the first is [ROCm/flash-attention](https://github.com/ROCm/flash-attention) based on a [Composable Kernel](https://github.com/ROCm/composable_kernel) (CK) implementation, and the second is a [Triton implementation](https://github.com/huggingface/text-generation-inference/blob/main/server/text_generation_server/layers/attention/flash_attn_triton.py).
 
 By default, the Composable Kernel implementation is used. However, the Triton implementation has slightly lower latency on MI250 and MI300, but requires a warmup which can be prohibitive as it needs to be done again for each new prompt length. If needed, FA Triton impelmentation can be enabled with `--env ROCM_USE_FLASH_ATTN_V2_TRITON="0"` when launching TGI's docker container.
 
+## Custom PagedAttention
+
+For better performance on ROCm, a custom Paged Attention kernel is available and is enabled by default. To disable it and fall back to the PagedAttention v2 kernel, set the environment variable `ROCM_USE_CUSTOM_PAGED_ATTN=0`.
+
+The custom kernel supports bf16 and fp16 data types, block size of 16, head size of 128, a maximum context length of 16k, and GQA ratios between 1 and 16. For other configurations, we use the PagedAttention v2 kernel.
+
 ## Unsupported features
 
 The following features are currently not supported in the ROCm version of TGI, and the supported may be extended in the future:
diff --git a/docs/source/installation_intel.md b/docs/source/installation_intel.md
new file mode 100644
index 00000000..1435b331
--- /dev/null
+++ b/docs/source/installation_intel.md
@@ -0,0 +1,36 @@
+# Using TGI with Intel GPUs
+
+TGI optimized models are supported on Intel Data Center GPU [Max1100](https://www.intel.com/content/www/us/en/products/sku/232876/intel-data-center-gpu-max-1100/specifications.html), [Max1550](https://www.intel.com/content/www/us/en/products/sku/232873/intel-data-center-gpu-max-1550/specifications.html), the recommended usage is through Docker.
+
+
+On a server powered by Intel GPUs, TGI can be launched with the following command:
+
+```bash
+model=teknium/OpenHermes-2.5-Mistral-7B
+volume=$PWD/data # share a volume with the Docker container to avoid downloading weights every run
+
+docker run --rm --privileged --cap-add=sys_nice \
+    --device=/dev/dri \
+    --ipc=host --shm-size 1g --net host -v $volume:/data \
+    ghcr.io/huggingface/text-generation-inference:2.3.1-intel-xpu \
+    --model-id $model --cuda-graphs 0
+```
+
+# Using TGI with Intel CPUs
+
+Intel® Extension for PyTorch (IPEX) also provides further optimizations for Intel CPUs. The IPEX provides optimization operations such as flash attention, page attention, Add + LayerNorm, ROPE and more.
+
+On a server powered by Intel CPU, TGI can be launched with the following command:
+
+```bash
+model=teknium/OpenHermes-2.5-Mistral-7B
+volume=$PWD/data # share a volume with the Docker container to avoid downloading weights every run
+
+docker run --rm --privileged --cap-add=sys_nice \
+    --device=/dev/dri \
+    --ipc=host --shm-size 1g --net host -v $volume:/data \
+    ghcr.io/huggingface/text-generation-inference:2.3.1-intel-cpu \
+    --model-id $model --cuda-graphs 0
+```
+
+The launched TGI server can then be queried from clients, make sure to check out the [Consuming TGI](./basic_tutorials/consuming_tgi) guide.
diff --git a/docs/source/installation_nvidia.md b/docs/source/installation_nvidia.md
index 62e1a3d6..634380fc 100644
--- a/docs/source/installation_nvidia.md
+++ b/docs/source/installation_nvidia.md
@@ -11,7 +11,7 @@ model=teknium/OpenHermes-2.5-Mistral-7B
 volume=$PWD/data # share a volume with the Docker container to avoid downloading weights every run
 
 docker run --gpus all --shm-size 64g -p 8080:80 -v $volume:/data \
-    ghcr.io/huggingface/text-generation-inference:2.0.3 \
+    ghcr.io/huggingface/text-generation-inference:2.3.1 \
     --model-id $model
 ```
 
diff --git a/docs/source/quicktour.md b/docs/source/quicktour.md
index 6137c6f6..655e6f9e 100644
--- a/docs/source/quicktour.md
+++ b/docs/source/quicktour.md
@@ -11,17 +11,26 @@ model=teknium/OpenHermes-2.5-Mistral-7B
 volume=$PWD/data # share a volume with the Docker container to avoid downloading weights every run
 
 docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data \
-    ghcr.io/huggingface/text-generation-inference:2.0.3 \
+    ghcr.io/huggingface/text-generation-inference:2.3.1 \
     --model-id $model
 ```
 
+<Tip>
+
+If you want to serve gated or private models, which provide
+controlled access to sensitive or proprietary content, refer to
+[this guide](https://huggingface.co/docs/text-generation-inference/en/basic_tutorials/gated_model_access)
+for detailed instructions.
+
+</Tip>
+
 ### Supported hardware
 
-TGI supports various hardware. Make sure to check the [Using TGI with Nvidia GPUs](./installation_nvidia), [Using TGI with AMD GPUs](./installation_amd), [Using TGI with Gaudi](./installation_gaudi), [Using TGI with Inferentia](./installation_inferentia) guides depending on which hardware you would like to deploy TGI on.
+TGI supports various hardware. Make sure to check the [Using TGI with Nvidia GPUs](./installation_nvidia), [Using TGI with AMD GPUs](./installation_amd), [Using TGI with Intel GPUs](./installation_intel), [Using TGI with Gaudi](./installation_gaudi), [Using TGI with Inferentia](./installation_inferentia) guides depending on which hardware you would like to deploy TGI on.
 
 ## Consuming TGI
 
-Once TGI is running, you can use the `generate` endpoint by doing requests. To learn more about how to query the endpoints, check the [Consuming TGI](./basic_tutorials/consuming_tgi) section, where we show examples with utility libraries and UIs. Below you can see a simple snippet to query the endpoint.
+Once TGI is running, you can use the `generate` endpoint or the Open AI Chat Completion API compatible [Messages API](https://huggingface.co/docs/text-generation-inference/en/messages_api) by doing requests. To learn more about how to query the endpoints, check the [Consuming TGI](./basic_tutorials/consuming_tgi) section, where we show examples with utility libraries and UIs. Below you can see a simple snippet to query the endpoint.
 
 <inferencesnippet>
 <python>
@@ -88,7 +97,7 @@ curl 127.0.0.1:8080/generate \
 To see all possible deploy flags and options, you can use the `--help` flag. It's possible to configure the number of shards, quantization, generation parameters, and more.
 
 ```bash
-docker run ghcr.io/huggingface/text-generation-inference:2.0.3 --help
+docker run ghcr.io/huggingface/text-generation-inference:2.2.0 --help
 ```
 
 </Tip>
diff --git a/docs/source/messages_api.md b/docs/source/reference/api_reference.md
similarity index 84%
rename from docs/source/messages_api.md
rename to docs/source/reference/api_reference.md
index 250aaae2..52043c80 100644
--- a/docs/source/messages_api.md
+++ b/docs/source/reference/api_reference.md
@@ -1,18 +1,31 @@
-# Messages API
+# HTTP API Reference
+
+#### Table of Contents
+
+- [Text Generation Inference custom API](#text-generation-inference-custom-api)
+- [OpenAI Messages API](#openai-messages-api)
+  - [Making a Request](#making-a-request)
+  - [Streaming](#streaming)
+  - [Synchronous](#synchronous)
+  - [Hugging Face Inference Endpoints](#hugging-face-inference-endpoints)
+  - [Cloud Providers](#cloud-providers)
+      - [Amazon SageMaker](#amazon-sagemaker)
+
+The HTTP API is a RESTful API that allows you to interact with the text-generation-inference component. Two endpoints are available:
+* Text Generation Inference [custom API](https://huggingface.github.io/text-generation-inference/)
+* OpenAI's [Messages API](#openai-messages-api)
+
+
+## Text Generation Inference custom API
+
+Check the [API documentation](https://huggingface.github.io/text-generation-inference/) for more information on how to interact with the Text Generation Inference API.
+
+## OpenAI Messages API
 
 Text Generation Inference (TGI) now supports the Messages API, which is fully compatible with the OpenAI Chat Completion API. This feature is available starting from version 1.4.0. You can use OpenAI's client libraries or third-party libraries expecting OpenAI schema to interact with TGI's Messages API. Below are some examples of how to utilize this compatibility.
 
 > **Note:** The Messages API is supported from TGI version 1.4.0 and above. Ensure you are using a compatible version to access this feature.
 
-#### Table of Contents
-
-- [Making a Request](#making-a-request)
-- [Streaming](#streaming)
-- [Synchronous](#synchronous)
-- [Hugging Face Inference Endpoints](#hugging-face-inference-endpoints)
-- [Cloud Providers](#cloud-providers)
-  - [Amazon SageMaker](#amazon-sagemaker)
-
 ## Making a Request
 
 You can make a request to TGI's Messages API using `curl`. Here's an example:
diff --git a/docs/source/basic_tutorials/launcher.md b/docs/source/reference/launcher.md
similarity index 89%
rename from docs/source/basic_tutorials/launcher.md
rename to docs/source/reference/launcher.md
index 1e5b6fd2..c8d2a4c6 100644
--- a/docs/source/basic_tutorials/launcher.md
+++ b/docs/source/reference/launcher.md
@@ -55,14 +55,18 @@ Options:
 ## QUANTIZE
 ```shell
       --quantize <QUANTIZE>
-          Whether you want the model to be quantized
+          Quantization method to use for the model. It is not necessary to specify this option for pre-quantized models, since the quantization method is read from the model configuration.
+          
+          Marlin kernels will be used automatically for GPTQ/AWQ models.
           
           [env: QUANTIZE=]
 
           Possible values:
           - awq:              4 bit quantization. Requires a specific AWQ quantized model: <https://hf.co/models?search=awq>. Should replace GPTQ models wherever possible because of the better latency
           - eetq:             8 bit quantization, doesn't require specific model. Should be a drop-in replacement to bitsandbytes with much better performance. Kernels are from <https://github.com/NetEase-FuXi/EETQ.git>
+          - exl2:             Variable bit quantization. Requires a specific EXL2 quantized model: <https://hf.co/models?search=exl2>. Requires exllama2 kernels and does not support tensor parallelism (num_shard > 1)
           - gptq:             4 bit quantization. Requires a specific GTPQ quantized model: <https://hf.co/models?search=gptq>. text-generation-inference will use exllama (faster) kernels wherever possible, and use triton kernel (wider support) when it's not. AWQ has faster kernels
+          - marlin:           4 bit quantization. Requires a specific Marlin quantized model: <https://hf.co/models?search=marlin>
           - bitsandbytes:     Bitsandbytes 8bit. Can be applied on any model, will cut the memory requirement in half, but it is known that the model will be much slower to run than the native f16
           - bitsandbytes-nf4: Bitsandbytes 4bit. Can be applied on any model, will cut the memory requirement by 4x, but it is known that the model will be much slower to run than the native f16
           - bitsandbytes-fp4: Bitsandbytes 4bit. nf4 should be preferred in most cases but maybe this one has better perplexity performance for you model
@@ -124,7 +128,7 @@ Options:
 ## MAX_TOP_N_TOKENS
 ```shell
       --max-top-n-tokens <MAX_TOP_N_TOKENS>
-          This is the maximum allowed value for clients to set `top_n_tokens`. `top_n_tokens is used to return information about the the `n` most likely tokens at each generation step, instead of just the sampled token. This information can be used for downstream tasks like for classification or ranking
+          This is the maximum allowed value for clients to set `top_n_tokens`. `top_n_tokens` is used to return information about the the `n` most likely tokens at each generation step, instead of just the sampled token. This information can be used for downstream tasks like for classification or ranking
           
           [env: MAX_TOP_N_TOKENS=]
           [default: 5]
@@ -334,12 +338,25 @@ Options:
       --otlp-endpoint <OTLP_ENDPOINT>
           [env: OTLP_ENDPOINT=]
 
+```
+## OTLP_SERVICE_NAME
+```shell
+      --otlp-service-name <OTLP_SERVICE_NAME>
+          [env: OTLP_SERVICE_NAME=]
+          [default: text-generation-inference.router]
+
 ```
 ## CORS_ALLOW_ORIGIN
 ```shell
       --cors-allow-origin <CORS_ALLOW_ORIGIN>
           [env: CORS_ALLOW_ORIGIN=]
 
+```
+## API_KEY
+```shell
+      --api-key <API_KEY>
+          [env: API_KEY=]
+
 ```
 ## WATERMARK_GAMMA
 ```shell
@@ -407,6 +424,28 @@ Options:
           [env: MAX_CLIENT_BATCH_SIZE=]
           [default: 4]
 
+```
+## LORA_ADAPTERS
+```shell
+      --lora-adapters <LORA_ADAPTERS>
+          Lora Adapters a list of adapter ids i.e. `repo/adapter1,repo/adapter2` to load during startup that will be available to callers via the `adapter_id` field in a request
+          
+          [env: LORA_ADAPTERS=]
+
+```
+## USAGE_STATS
+```shell
+      --usage-stats <USAGE_STATS>
+          Control if anonymous usage stats are collected. Options are "on", "off" and "no-stack" Defaul is on
+          
+          [env: USAGE_STATS=]
+          [default: on]
+
+          Possible values:
+          - on:       Default option, usage statistics are collected anonymously
+          - off:      Disables all collection of usage statistics
+          - no-stack: Doesn't send the error stack trace or error type, but allows sending a crash event
+
 ```
 ## HELP
 ```shell
diff --git a/docs/source/reference/metrics.md b/docs/source/reference/metrics.md
new file mode 100644
index 00000000..d34d38ea
--- /dev/null
+++ b/docs/source/reference/metrics.md
@@ -0,0 +1,30 @@
+# Metrics
+
+TGI exposes multiple metrics that can be collected via the `/metrics` Prometheus endpoint.
+These metrics can be used to monitor the performance of TGI, autoscale deployment and to help identify bottlenecks.
+
+The following metrics are exposed:
+
+| Metric Name                                | Description                                                                              | Type      | Unit    |
+|--------------------------------------------|------------------------------------------------------------------------------------------|-----------|---------|
+| `tgi_batch_current_max_tokens`             | Maximum tokens for the current batch                                                     | Gauge     | Count   |
+| `tgi_batch_current_size`                   | Current batch size                                                                       | Gauge     | Count   |
+| `tgi_batch_decode_duration`                | Time spent decoding a batch per method (prefill or decode)                               | Histogram | Seconds |
+| `tgi_batch_filter_duration`                | Time spent filtering batches and sending generated tokens per method (prefill or decode) | Histogram | Seconds |
+| `tgi_batch_forward_duration`               | Batch forward duration per method (prefill or decode)                                    | Histogram | Seconds |
+| `tgi_batch_inference_count`                | Inference calls per method (prefill or decode)                                           | Counter   | Count   |
+| `tgi_batch_inference_duration`             | Batch inference duration                                                                 | Histogram | Seconds |
+| `tgi_batch_inference_success`              | Number of successful inference calls per method (prefill or decode)                      | Counter   | Count   |
+| `tgi_batch_next_size`                      | Batch size of the next batch                                                             | Histogram | Count   |
+| `tgi_queue_size`                           | Current queue size                                                                       | Gauge     | Count   |
+| `tgi_request_count`                        | Total number of requests                                                                 | Counter   | Count   |
+| `tgi_request_duration`                     | Total time spent processing the request (e2e latency)                                    | Histogram | Seconds |
+| `tgi_request_generated_tokens`             | Generated tokens per request                                                             | Histogram | Count   |
+| `tgi_request_inference_duration`           | Request inference duration                                                               | Histogram | Seconds |
+| `tgi_request_input_length`                 | Input token length per request                                                           | Histogram | Count   |
+| `tgi_request_max_new_tokens`               | Maximum new tokens per request                                                           | Histogram | Count   |
+| `tgi_request_mean_time_per_token_duration` | Mean time per token per request (inter-token latency)                                    | Histogram | Seconds |
+| `tgi_request_queue_duration`               | Time spent in the queue per request                                                      | Histogram | Seconds |
+| `tgi_request_skipped_tokens`               | Speculated tokens per request                                                            | Histogram | Count   |
+| `tgi_request_success`                      | Number of successful requests                                                            | Counter   |         |
+| `tgi_request_validation_duration`          | Time spent validating the request                                                        | Histogram | Seconds |
diff --git a/docs/source/supported_models.md b/docs/source/supported_models.md
index 4b6cf731..be280a2b 100644
--- a/docs/source/supported_models.md
+++ b/docs/source/supported_models.md
@@ -1,26 +1,30 @@
 
 # Supported Models and Hardware
 
-Text Generation Inference enables serving optimized models on specific hardware for the highest performance. The following sections list which models are hardware are supported.
+Text Generation Inference enables serving optimized models on specific hardware for the highest performance. The following sections list which models (VLMs & LLMs) are supported.
 
 ## Supported Models
 
+- [Deepseek V2](https://huggingface.co/deepseek-ai/DeepSeek-V2)
 - [Idefics 2](https://huggingface.co/HuggingFaceM4/idefics2-8b) (Multimodal)
 - [Llava Next (1.6)](https://huggingface.co/llava-hf/llava-v1.6-vicuna-13b-hf) (Multimodal)
-- [Llama](https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct)
+- [Llama](https://huggingface.co/collections/meta-llama/llama-31-669fc079a0c406a149a5738f)
 - [Phi 3](https://huggingface.co/microsoft/Phi-3-mini-4k-instruct)
 - [Gemma](https://huggingface.co/google/gemma-7b)
+- [PaliGemma](https://huggingface.co/google/paligemma-3b-pt-224)
+- [Gemma2](https://huggingface.co/collections/google/gemma-2-release-667d6600fd5220e7b967f315)
 - [Cohere](https://huggingface.co/CohereForAI/c4ai-command-r-plus)
 - [Dbrx](https://huggingface.co/databricks/dbrx-instruct)
 - [Mamba](https://huggingface.co/state-spaces/mamba-2.8b-slimpj)
-- [Mistral](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2)
+- [Mistral](https://huggingface.co/mistralai/Mistral-Nemo-Instruct-2407)
 - [Mixtral](https://huggingface.co/mistralai/Mixtral-8x22B-Instruct-v0.1)
 - [Gpt Bigcode](https://huggingface.co/bigcode/gpt_bigcode-santacoder)
 - [Phi](https://huggingface.co/microsoft/phi-1_5)
+- [PhiMoe](https://huggingface.co/microsoft/Phi-3.5-MoE-instruct)
 - [Baichuan](https://huggingface.co/baichuan-inc/Baichuan2-7B-Chat)
 - [Falcon](https://huggingface.co/tiiuae/falcon-7b-instruct)
 - [StarCoder 2](https://huggingface.co/bigcode/starcoder2-15b-instruct-v0.1)
-- [Qwen 2](https://huggingface.co/bigcode/starcoder2-15b-instruct-v0.1)
+- [Qwen 2](https://huggingface.co/collections/Qwen/qwen2-6659360b33528ced941e557f)
 - [Opt](https://huggingface.co/facebook/opt-6.7b)
 - [T5](https://huggingface.co/google/flan-t5-xxl)
 - [Galactica](https://huggingface.co/facebook/galactica-120b)
@@ -29,7 +33,9 @@ Text Generation Inference enables serving optimized models on specific hardware
 - [Mpt](https://huggingface.co/mosaicml/mpt-7b-instruct)
 - [Gpt2](https://huggingface.co/openai-community/gpt2)
 - [Gpt Neox](https://huggingface.co/EleutherAI/gpt-neox-20b)
+- [Gptj](https://huggingface.co/EleutherAI/gpt-j-6b)
 - [Idefics](https://huggingface.co/HuggingFaceM4/idefics-9b) (Multimodal)
+- [Mllama](https://huggingface.co/meta-llama/Llama-3.2-11B-Vision-Instruct) (Multimodal)
 
 
 If the above list lacks the model you would like to serve, depending on the model's pipeline type, you can try to initialize and serve the model anyways to see how well it performs, but performance isn't guaranteed for non-optimized models:
diff --git a/docs/source/usage_statistics.md b/docs/source/usage_statistics.md
new file mode 100644
index 00000000..a2c406ec
--- /dev/null
+++ b/docs/source/usage_statistics.md
@@ -0,0 +1,75 @@
+
+# Collection of Usage Statistics
+
+Text Generation Inference collects anonymous usage statistics to help us improve the service. The collected data is used to improve TGI and to understand what causes failures. The data is collected transparently and any sensitive information is omitted.
+
+Data is sent twice, once on server startup and once when server stops. Also, usage statistics are only enabled when TGI is running in docker to avoid collecting data then TGI runs directly on the host machine.
+
+## What data is collected
+
+The code that collects the data is available [here](https://github.com/huggingface/text-generation-inference/blob/main/router/src/usage_stats.rs).
+As of release 2.1.2 this is an example of the data collected:
+
+- From the TGI configuration:
+```json
+{
+  "event_type": "start",
+  "disable_grammar_support": false,
+  "max_batch_prefill_tokens": 4096,
+  "max_batch_size": null,
+  "max_batch_total_tokens": null,
+  "max_best_of": 2,
+  "max_client_batch_size": 4,
+  "max_concurrent_requests": 128,
+  "max_input_tokens": 1024,
+  "max_stop_sequences": 4,
+  "max_top_n_tokens": 5,
+  "max_total_tokens": 2048,
+  "max_waiting_tokens": 20,
+  "messages_api_enabled": false,
+  "model_config": {
+    "model_type": "Bloom"
+  },
+  "revision": null,
+  "tokenizer_class": "BloomTokenizerFast",
+  "validation_workers": 2,
+  "waiting_served_ratio": 1.2,
+  "docker_label": "latest",
+  "git_sha": "cfc118704880453d29bcbe4fbbd91dda501cf5fe",
+  "nvidia_env": {
+    "name": "NVIDIA A10G",
+    "pci_bus_id": "00000000:00:1E.0",
+    "driver_version": "535.183.01",
+    "pstate": "P8",
+    "pcie_link_gen_max": "4",
+    "pcie_link_gen_current": "1",
+    "temperature_gpu": "31",
+    "utilization_gpu": "0 %",
+    "utilization_memory": "0 %",
+    "memory_total": "23028 MiB",
+    "memory_free": "22515 MiB",
+    "memory_used": "0 MiB",
+    "reset_status_reset_required": "No",
+    "reset_status_drain_and_reset_recommended": "No",
+    "compute_cap": "8.6",
+    "ecc_errors_corrected_volatile_total": "0",
+    "mig_mode_current": "[N/A]",
+    "power_draw_instant": "10.86 W",
+    "power_limit": "300.00 W"
+  },
+  "system_env": {
+    "cpu_count": 16,
+    "cpu_type": "AMD EPYC 7R32",
+    "total_memory": 66681196544,
+    "architecture": "x86_64",
+    "platform": "linux-unix-x86_64"
+  }
+}
+
+```
+
+## How to opt-out
+
+By passing the `--usage-stats` to the text-generation-launcher you can control how much usage statistics are being collected.
+`--usage-stats=no-stack` will not emit the stack traces from errors and the error types, but will continue to send start and stop events
+`--usage-stats=off` will completely disable everything
diff --git a/examples/run_generation.py b/examples/run_generation.py
index d37eed4e..a2b43e6f 100644
--- a/examples/run_generation.py
+++ b/examples/run_generation.py
@@ -14,7 +14,7 @@ from tgi_client import TgiClient
 def get_args():
     parser = argparse.ArgumentParser()
     parser.add_argument(
-        "--server_address", type=str, default="http://localhost:8080", help="Address of the TGI server"
+        "--server_address", type=str, default="http://localhost:8083", help="Address of the TGI server"
     )
     parser.add_argument(
         "--model_id", type=str, default="meta-llama/Llama-2-7b-chat-hf", help="Model id used in TGI server"
@@ -56,6 +56,7 @@ def read_dataset(
     )
     if len(dataset) > total_sample_count:
         dataset = dataset.select(range(total_sample_count))
+
     dataset = dataset.shuffle(seed=seed)
     return [sample["prompt"] for sample in dataset]
 
diff --git a/flake.lock b/flake.lock
new file mode 100644
index 00000000..30b11d5b
--- /dev/null
+++ b/flake.lock
@@ -0,0 +1,997 @@
+{
+  "nodes": {
+    "cachix": {
+      "inputs": {
+        "devenv": [
+          "crate2nix"
+        ],
+        "flake-compat": [
+          "crate2nix"
+        ],
+        "nixpkgs": "nixpkgs",
+        "pre-commit-hooks": [
+          "crate2nix"
+        ]
+      },
+      "locked": {
+        "lastModified": 1709700175,
+        "narHash": "sha256-A0/6ZjLmT9qdYzKHmevnEIC7G+GiZ4UCr8v0poRPzds=",
+        "owner": "cachix",
+        "repo": "cachix",
+        "rev": "be97b37989f11b724197b5f4c7ffd78f12c8c4bf",
+        "type": "github"
+      },
+      "original": {
+        "owner": "cachix",
+        "ref": "latest",
+        "repo": "cachix",
+        "type": "github"
+      }
+    },
+    "cachix_2": {
+      "inputs": {
+        "devenv": [
+          "crate2nix",
+          "crate2nix_stable"
+        ],
+        "flake-compat": [
+          "crate2nix",
+          "crate2nix_stable"
+        ],
+        "nixpkgs": "nixpkgs_2",
+        "pre-commit-hooks": [
+          "crate2nix",
+          "crate2nix_stable"
+        ]
+      },
+      "locked": {
+        "lastModified": 1716549461,
+        "narHash": "sha256-lHy5kgx6J8uD+16SO47dPrbob98sh+W1tf4ceSqPVK4=",
+        "owner": "cachix",
+        "repo": "cachix",
+        "rev": "e2bb269fb8c0828d5d4d2d7b8d09ea85abcacbd4",
+        "type": "github"
+      },
+      "original": {
+        "owner": "cachix",
+        "ref": "latest",
+        "repo": "cachix",
+        "type": "github"
+      }
+    },
+    "cachix_3": {
+      "inputs": {
+        "devenv": [
+          "crate2nix",
+          "crate2nix_stable",
+          "crate2nix_stable"
+        ],
+        "flake-compat": [
+          "crate2nix",
+          "crate2nix_stable",
+          "crate2nix_stable"
+        ],
+        "nixpkgs": "nixpkgs_3",
+        "pre-commit-hooks": [
+          "crate2nix",
+          "crate2nix_stable",
+          "crate2nix_stable"
+        ]
+      },
+      "locked": {
+        "lastModified": 1716549461,
+        "narHash": "sha256-lHy5kgx6J8uD+16SO47dPrbob98sh+W1tf4ceSqPVK4=",
+        "owner": "cachix",
+        "repo": "cachix",
+        "rev": "e2bb269fb8c0828d5d4d2d7b8d09ea85abcacbd4",
+        "type": "github"
+      },
+      "original": {
+        "owner": "cachix",
+        "ref": "latest",
+        "repo": "cachix",
+        "type": "github"
+      }
+    },
+    "crate2nix": {
+      "inputs": {
+        "cachix": "cachix",
+        "crate2nix_stable": "crate2nix_stable",
+        "devshell": "devshell_3",
+        "flake-compat": "flake-compat_3",
+        "flake-parts": "flake-parts_3",
+        "nix-test-runner": "nix-test-runner_3",
+        "nixpkgs": [
+          "tgi-nix",
+          "nixpkgs"
+        ],
+        "pre-commit-hooks": "pre-commit-hooks_3"
+      },
+      "locked": {
+        "lastModified": 1723311214,
+        "narHash": "sha256-xdGZQBEa1AC2us/sY3igS/CucWY6jErXsAvCFRhB2LI=",
+        "owner": "nix-community",
+        "repo": "crate2nix",
+        "rev": "236f6addfd452a48be805819e3216af79e988fd5",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "crate2nix",
+        "type": "github"
+      }
+    },
+    "crate2nix_stable": {
+      "inputs": {
+        "cachix": "cachix_2",
+        "crate2nix_stable": "crate2nix_stable_2",
+        "devshell": "devshell_2",
+        "flake-compat": "flake-compat_2",
+        "flake-parts": "flake-parts_2",
+        "nix-test-runner": "nix-test-runner_2",
+        "nixpkgs": "nixpkgs_5",
+        "pre-commit-hooks": "pre-commit-hooks_2"
+      },
+      "locked": {
+        "lastModified": 1719760004,
+        "narHash": "sha256-esWhRnt7FhiYq0CcIxw9pvH+ybOQmWBfHYMtleaMhBE=",
+        "owner": "nix-community",
+        "repo": "crate2nix",
+        "rev": "1dee214bb20855fa3e1e7bb98d28922ddaff8c57",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "ref": "0.14.1",
+        "repo": "crate2nix",
+        "type": "github"
+      }
+    },
+    "crate2nix_stable_2": {
+      "inputs": {
+        "cachix": "cachix_3",
+        "crate2nix_stable": "crate2nix_stable_3",
+        "devshell": "devshell",
+        "flake-compat": "flake-compat",
+        "flake-parts": "flake-parts",
+        "nix-test-runner": "nix-test-runner",
+        "nixpkgs": "nixpkgs_4",
+        "pre-commit-hooks": "pre-commit-hooks"
+      },
+      "locked": {
+        "lastModified": 1712821484,
+        "narHash": "sha256-rGT3CW64cJS9nlnWPFWSc1iEa3dNZecVVuPVGzcsHe8=",
+        "owner": "nix-community",
+        "repo": "crate2nix",
+        "rev": "42883afcad3823fa5811e967fb7bff54bc3c9d6d",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "ref": "0.14.0",
+        "repo": "crate2nix",
+        "type": "github"
+      }
+    },
+    "crate2nix_stable_3": {
+      "inputs": {
+        "flake-utils": "flake-utils"
+      },
+      "locked": {
+        "lastModified": 1702842982,
+        "narHash": "sha256-A9AowkHIjsy1a4LuiPiVP88FMxyCWK41flZEZOUuwQM=",
+        "owner": "nix-community",
+        "repo": "crate2nix",
+        "rev": "75ac2973affa6b9b4f661a7b592cba6e4f51d426",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "ref": "0.12.0",
+        "repo": "crate2nix",
+        "type": "github"
+      }
+    },
+    "devshell": {
+      "inputs": {
+        "flake-utils": "flake-utils_2",
+        "nixpkgs": [
+          "crate2nix",
+          "crate2nix_stable",
+          "crate2nix_stable",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1717408969,
+        "narHash": "sha256-Q0OEFqe35fZbbRPPRdrjTUUChKVhhWXz3T9ZSKmaoVY=",
+        "owner": "numtide",
+        "repo": "devshell",
+        "rev": "1ebbe68d57457c8cae98145410b164b5477761f4",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "devshell",
+        "type": "github"
+      }
+    },
+    "devshell_2": {
+      "inputs": {
+        "flake-utils": "flake-utils_3",
+        "nixpkgs": [
+          "crate2nix",
+          "crate2nix_stable",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1717408969,
+        "narHash": "sha256-Q0OEFqe35fZbbRPPRdrjTUUChKVhhWXz3T9ZSKmaoVY=",
+        "owner": "numtide",
+        "repo": "devshell",
+        "rev": "1ebbe68d57457c8cae98145410b164b5477761f4",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "devshell",
+        "type": "github"
+      }
+    },
+    "devshell_3": {
+      "inputs": {
+        "flake-utils": "flake-utils_4",
+        "nixpkgs": [
+          "crate2nix",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1711099426,
+        "narHash": "sha256-HzpgM/wc3aqpnHJJ2oDqPBkNsqWbW0WfWUO8lKu8nGk=",
+        "owner": "numtide",
+        "repo": "devshell",
+        "rev": "2d45b54ca4a183f2fdcf4b19c895b64fbf620ee8",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "devshell",
+        "type": "github"
+      }
+    },
+    "flake-compat": {
+      "locked": {
+        "lastModified": 1696426674,
+        "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=",
+        "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33",
+        "revCount": 57,
+        "type": "tarball",
+        "url": "https://api.flakehub.com/f/pinned/edolstra/flake-compat/1.0.1/018afb31-abd1-7bff-a5e4-cff7e18efb7a/source.tar.gz"
+      },
+      "original": {
+        "type": "tarball",
+        "url": "https://flakehub.com/f/edolstra/flake-compat/1.tar.gz"
+      }
+    },
+    "flake-compat_2": {
+      "locked": {
+        "lastModified": 1696426674,
+        "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=",
+        "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33",
+        "revCount": 57,
+        "type": "tarball",
+        "url": "https://api.flakehub.com/f/pinned/edolstra/flake-compat/1.0.1/018afb31-abd1-7bff-a5e4-cff7e18efb7a/source.tar.gz"
+      },
+      "original": {
+        "type": "tarball",
+        "url": "https://flakehub.com/f/edolstra/flake-compat/1.tar.gz"
+      }
+    },
+    "flake-compat_3": {
+      "locked": {
+        "lastModified": 1696426674,
+        "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=",
+        "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33",
+        "revCount": 57,
+        "type": "tarball",
+        "url": "https://api.flakehub.com/f/pinned/edolstra/flake-compat/1.0.1/018afb31-abd1-7bff-a5e4-cff7e18efb7a/source.tar.gz"
+      },
+      "original": {
+        "type": "tarball",
+        "url": "https://flakehub.com/f/edolstra/flake-compat/1.tar.gz"
+      }
+    },
+    "flake-compat_4": {
+      "locked": {
+        "lastModified": 1696426674,
+        "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=",
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
+    "flake-parts": {
+      "inputs": {
+        "nixpkgs-lib": [
+          "crate2nix",
+          "crate2nix_stable",
+          "crate2nix_stable",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1719745305,
+        "narHash": "sha256-xwgjVUpqSviudEkpQnioeez1Uo2wzrsMaJKJClh+Bls=",
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "rev": "c3c5ecc05edc7dafba779c6c1a61cd08ac6583e9",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "type": "github"
+      }
+    },
+    "flake-parts_2": {
+      "inputs": {
+        "nixpkgs-lib": [
+          "crate2nix",
+          "crate2nix_stable",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1719745305,
+        "narHash": "sha256-xwgjVUpqSviudEkpQnioeez1Uo2wzrsMaJKJClh+Bls=",
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "rev": "c3c5ecc05edc7dafba779c6c1a61cd08ac6583e9",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "type": "github"
+      }
+    },
+    "flake-parts_3": {
+      "inputs": {
+        "nixpkgs-lib": [
+          "crate2nix",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1712014858,
+        "narHash": "sha256-sB4SWl2lX95bExY2gMFG5HIzvva5AVMJd4Igm+GpZNw=",
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "rev": "9126214d0a59633752a136528f5f3b9aa8565b7d",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "type": "github"
+      }
+    },
+    "flake-utils": {
+      "inputs": {
+        "systems": "systems"
+      },
+      "locked": {
+        "lastModified": 1694529238,
+        "narHash": "sha256-zsNZZGTGnMOf9YpHKJqMSsa0dXbfmxeoJ7xHlrt+xmY=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "ff7b65b44d01cf9ba6a71320833626af21126384",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "flake-utils_2": {
+      "inputs": {
+        "systems": "systems_2"
+      },
+      "locked": {
+        "lastModified": 1701680307,
+        "narHash": "sha256-kAuep2h5ajznlPMD9rnQyffWG8EM/C73lejGofXvdM8=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "4022d587cbbfd70fe950c1e2083a02621806a725",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "flake-utils_3": {
+      "inputs": {
+        "systems": "systems_3"
+      },
+      "locked": {
+        "lastModified": 1701680307,
+        "narHash": "sha256-kAuep2h5ajznlPMD9rnQyffWG8EM/C73lejGofXvdM8=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "4022d587cbbfd70fe950c1e2083a02621806a725",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "flake-utils_4": {
+      "inputs": {
+        "systems": "systems_4"
+      },
+      "locked": {
+        "lastModified": 1701680307,
+        "narHash": "sha256-kAuep2h5ajznlPMD9rnQyffWG8EM/C73lejGofXvdM8=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "4022d587cbbfd70fe950c1e2083a02621806a725",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "flake-utils_5": {
+      "inputs": {
+        "systems": "systems_5"
+      },
+      "locked": {
+        "lastModified": 1710146030,
+        "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "flake-utils_6": {
+      "inputs": {
+        "systems": "systems_6"
+      },
+      "locked": {
+        "lastModified": 1726560853,
+        "narHash": "sha256-X6rJYSESBVr3hBoH0WbKE5KvhPU5bloyZ2L4K60/fPQ=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "c1dfcf08411b08f6b8615f7d8971a2bfa81d5e8a",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "flake-utils_7": {
+      "inputs": {
+        "systems": "systems_7"
+      },
+      "locked": {
+        "lastModified": 1726560853,
+        "narHash": "sha256-X6rJYSESBVr3hBoH0WbKE5KvhPU5bloyZ2L4K60/fPQ=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "c1dfcf08411b08f6b8615f7d8971a2bfa81d5e8a",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "gitignore": {
+      "inputs": {
+        "nixpkgs": [
+          "crate2nix",
+          "crate2nix_stable",
+          "crate2nix_stable",
+          "pre-commit-hooks",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1709087332,
+        "narHash": "sha256-HG2cCnktfHsKV0s4XW83gU3F57gaTljL9KNSuG6bnQs=",
+        "owner": "hercules-ci",
+        "repo": "gitignore.nix",
+        "rev": "637db329424fd7e46cf4185293b9cc8c88c95394",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "gitignore.nix",
+        "type": "github"
+      }
+    },
+    "gitignore_2": {
+      "inputs": {
+        "nixpkgs": [
+          "crate2nix",
+          "crate2nix_stable",
+          "pre-commit-hooks",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1709087332,
+        "narHash": "sha256-HG2cCnktfHsKV0s4XW83gU3F57gaTljL9KNSuG6bnQs=",
+        "owner": "hercules-ci",
+        "repo": "gitignore.nix",
+        "rev": "637db329424fd7e46cf4185293b9cc8c88c95394",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "gitignore.nix",
+        "type": "github"
+      }
+    },
+    "gitignore_3": {
+      "inputs": {
+        "nixpkgs": [
+          "crate2nix",
+          "pre-commit-hooks",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1709087332,
+        "narHash": "sha256-HG2cCnktfHsKV0s4XW83gU3F57gaTljL9KNSuG6bnQs=",
+        "owner": "hercules-ci",
+        "repo": "gitignore.nix",
+        "rev": "637db329424fd7e46cf4185293b9cc8c88c95394",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "gitignore.nix",
+        "type": "github"
+      }
+    },
+    "nix-filter": {
+      "locked": {
+        "lastModified": 1710156097,
+        "narHash": "sha256-1Wvk8UP7PXdf8bCCaEoMnOT1qe5/Duqgj+rL8sRQsSM=",
+        "owner": "numtide",
+        "repo": "nix-filter",
+        "rev": "3342559a24e85fc164b295c3444e8a139924675b",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "nix-filter",
+        "type": "github"
+      }
+    },
+    "nix-test-runner": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1588761593,
+        "narHash": "sha256-FKJykltAN/g3eIceJl4SfDnnyuH2jHImhMrXS2KvGIs=",
+        "owner": "stoeffel",
+        "repo": "nix-test-runner",
+        "rev": "c45d45b11ecef3eb9d834c3b6304c05c49b06ca2",
+        "type": "github"
+      },
+      "original": {
+        "owner": "stoeffel",
+        "repo": "nix-test-runner",
+        "type": "github"
+      }
+    },
+    "nix-test-runner_2": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1588761593,
+        "narHash": "sha256-FKJykltAN/g3eIceJl4SfDnnyuH2jHImhMrXS2KvGIs=",
+        "owner": "stoeffel",
+        "repo": "nix-test-runner",
+        "rev": "c45d45b11ecef3eb9d834c3b6304c05c49b06ca2",
+        "type": "github"
+      },
+      "original": {
+        "owner": "stoeffel",
+        "repo": "nix-test-runner",
+        "type": "github"
+      }
+    },
+    "nix-test-runner_3": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1588761593,
+        "narHash": "sha256-FKJykltAN/g3eIceJl4SfDnnyuH2jHImhMrXS2KvGIs=",
+        "owner": "stoeffel",
+        "repo": "nix-test-runner",
+        "rev": "c45d45b11ecef3eb9d834c3b6304c05c49b06ca2",
+        "type": "github"
+      },
+      "original": {
+        "owner": "stoeffel",
+        "repo": "nix-test-runner",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1700612854,
+        "narHash": "sha256-yrQ8osMD+vDLGFX7pcwsY/Qr5PUd6OmDMYJZzZi0+zc=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "19cbff58383a4ae384dea4d1d0c823d72b49d614",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs_2": {
+      "locked": {
+        "lastModified": 1715534503,
+        "narHash": "sha256-5ZSVkFadZbFP1THataCaSf0JH2cAH3S29hU9rrxTEqk=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "2057814051972fa1453ddfb0d98badbea9b83c06",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs_3": {
+      "locked": {
+        "lastModified": 1715534503,
+        "narHash": "sha256-5ZSVkFadZbFP1THataCaSf0JH2cAH3S29hU9rrxTEqk=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "2057814051972fa1453ddfb0d98badbea9b83c06",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs_4": {
+      "locked": {
+        "lastModified": 1719506693,
+        "narHash": "sha256-C8e9S7RzshSdHB7L+v9I51af1gDM5unhJ2xO1ywxNH8=",
+        "path": "/nix/store/4p0avw1s3vf27hspgqsrqs37gxk4i83i-source",
+        "rev": "b2852eb9365c6de48ffb0dc2c9562591f652242a",
+        "type": "path"
+      },
+      "original": {
+        "id": "nixpkgs",
+        "type": "indirect"
+      }
+    },
+    "nixpkgs_5": {
+      "locked": {
+        "lastModified": 1719506693,
+        "narHash": "sha256-C8e9S7RzshSdHB7L+v9I51af1gDM5unhJ2xO1ywxNH8=",
+        "path": "/nix/store/4p0avw1s3vf27hspgqsrqs37gxk4i83i-source",
+        "rev": "b2852eb9365c6de48ffb0dc2c9562591f652242a",
+        "type": "path"
+      },
+      "original": {
+        "id": "nixpkgs",
+        "type": "indirect"
+      }
+    },
+    "nixpkgs_6": {
+      "locked": {
+        "lastModified": 1727675176,
+        "narHash": "sha256-xIjBFMYldWvj+g8ahxMPofsj+OqxvKJN6YylNHQ7gn4=",
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "rev": "a6d0207fea9212d28cd3d487efe6bc699663b93a",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nixos",
+        "ref": "nixos-unstable-small",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "pre-commit-hooks": {
+      "inputs": {
+        "flake-compat": [
+          "crate2nix",
+          "crate2nix_stable",
+          "crate2nix_stable",
+          "flake-compat"
+        ],
+        "gitignore": "gitignore",
+        "nixpkgs": [
+          "crate2nix",
+          "crate2nix_stable",
+          "crate2nix_stable",
+          "nixpkgs"
+        ],
+        "nixpkgs-stable": [
+          "crate2nix",
+          "crate2nix_stable",
+          "crate2nix_stable",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1719259945,
+        "narHash": "sha256-F1h+XIsGKT9TkGO3omxDLEb/9jOOsI6NnzsXFsZhry4=",
+        "owner": "cachix",
+        "repo": "pre-commit-hooks.nix",
+        "rev": "0ff4381bbb8f7a52ca4a851660fc7a437a4c6e07",
+        "type": "github"
+      },
+      "original": {
+        "owner": "cachix",
+        "repo": "pre-commit-hooks.nix",
+        "type": "github"
+      }
+    },
+    "pre-commit-hooks_2": {
+      "inputs": {
+        "flake-compat": [
+          "crate2nix",
+          "crate2nix_stable",
+          "flake-compat"
+        ],
+        "gitignore": "gitignore_2",
+        "nixpkgs": [
+          "crate2nix",
+          "crate2nix_stable",
+          "nixpkgs"
+        ],
+        "nixpkgs-stable": [
+          "crate2nix",
+          "crate2nix_stable",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1719259945,
+        "narHash": "sha256-F1h+XIsGKT9TkGO3omxDLEb/9jOOsI6NnzsXFsZhry4=",
+        "owner": "cachix",
+        "repo": "pre-commit-hooks.nix",
+        "rev": "0ff4381bbb8f7a52ca4a851660fc7a437a4c6e07",
+        "type": "github"
+      },
+      "original": {
+        "owner": "cachix",
+        "repo": "pre-commit-hooks.nix",
+        "type": "github"
+      }
+    },
+    "pre-commit-hooks_3": {
+      "inputs": {
+        "flake-compat": [
+          "crate2nix",
+          "flake-compat"
+        ],
+        "flake-utils": "flake-utils_5",
+        "gitignore": "gitignore_3",
+        "nixpkgs": [
+          "crate2nix",
+          "nixpkgs"
+        ],
+        "nixpkgs-stable": [
+          "crate2nix",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1712055707,
+        "narHash": "sha256-4XLvuSIDZJGS17xEwSrNuJLL7UjDYKGJSbK1WWX2AK8=",
+        "owner": "cachix",
+        "repo": "pre-commit-hooks.nix",
+        "rev": "e35aed5fda3cc79f88ed7f1795021e559582093a",
+        "type": "github"
+      },
+      "original": {
+        "owner": "cachix",
+        "repo": "pre-commit-hooks.nix",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "crate2nix": "crate2nix",
+        "flake-utils": "flake-utils_6",
+        "nix-filter": "nix-filter",
+        "nixpkgs": [
+          "tgi-nix",
+          "nixpkgs"
+        ],
+        "rust-overlay": "rust-overlay",
+        "tgi-nix": "tgi-nix"
+      }
+    },
+    "rust-overlay": {
+      "inputs": {
+        "nixpkgs": [
+          "tgi-nix",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1727836133,
+        "narHash": "sha256-JE0zciM5IGWvK8J/pE2VldNBf7oyMH5WrU8tZArefbg=",
+        "owner": "oxalica",
+        "repo": "rust-overlay",
+        "rev": "02321540b0c8000b36889b1b974d1fec585b25a4",
+        "type": "github"
+      },
+      "original": {
+        "owner": "oxalica",
+        "repo": "rust-overlay",
+        "type": "github"
+      }
+    },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
+    "systems_2": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
+    "systems_3": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
+    "systems_4": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
+    "systems_5": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
+    "systems_6": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
+    "systems_7": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
+    "tgi-nix": {
+      "inputs": {
+        "flake-compat": "flake-compat_4",
+        "flake-utils": "flake-utils_7",
+        "nixpkgs": "nixpkgs_6"
+      },
+      "locked": {
+        "lastModified": 1727859277,
+        "narHash": "sha256-AsrPuQqhg8x5RRR3aX0vvDDRQb+HREq2wGxXOpZnWus=",
+        "owner": "huggingface",
+        "repo": "text-generation-inference-nix",
+        "rev": "14196ab62f31d005f46207f7a251f82a81d0a09f",
+        "type": "github"
+      },
+      "original": {
+        "owner": "huggingface",
+        "repo": "text-generation-inference-nix",
+        "type": "github"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}
diff --git a/flake.nix b/flake.nix
new file mode 100644
index 00000000..22fa8cfc
--- /dev/null
+++ b/flake.nix
@@ -0,0 +1,167 @@
+{
+  inputs = {
+    crate2nix = {
+      url = "github:nix-community/crate2nix";
+      inputs.nixpkgs.follows = "tgi-nix/nixpkgs";
+    };
+    nix-filter.url = "github:numtide/nix-filter";
+    tgi-nix.url = "github:huggingface/text-generation-inference-nix";
+    nixpkgs.follows = "tgi-nix/nixpkgs";
+    flake-utils.url = "github:numtide/flake-utils";
+    rust-overlay = {
+      url = "github:oxalica/rust-overlay";
+      inputs.nixpkgs.follows = "tgi-nix/nixpkgs";
+    };
+  };
+  outputs =
+    {
+      self,
+      crate2nix,
+      nix-filter,
+      nixpkgs,
+      flake-utils,
+      rust-overlay,
+      tgi-nix,
+    }:
+    flake-utils.lib.eachDefaultSystem (
+      system:
+      let
+        cargoNix = crate2nix.tools.${system}.appliedCargoNix {
+          name = "tgi";
+          src = ./.;
+          additionalCargoNixArgs = [ "--all-features" ];
+        };
+        pkgs = import nixpkgs {
+          inherit system;
+          inherit (tgi-nix.lib) config;
+          overlays = [
+            rust-overlay.overlays.default
+            tgi-nix.overlays.default
+          ];
+        };
+        crateOverrides = import ./nix/crate-overrides.nix { inherit pkgs nix-filter; };
+        benchmark = cargoNix.workspaceMembers.text-generation-benchmark.build.override {
+          inherit crateOverrides;
+        };
+        launcher = cargoNix.workspaceMembers.text-generation-launcher.build.override {
+          inherit crateOverrides;
+        };
+        router =
+          let
+            routerUnwrapped = cargoNix.workspaceMembers.text-generation-router-v3.build.override {
+              inherit crateOverrides;
+            };
+            packagePath =
+              with pkgs.python3.pkgs;
+              makePythonPath [
+                protobuf
+                sentencepiece
+                torch
+                transformers
+              ];
+          in
+          pkgs.writeShellApplication {
+            name = "text-generation-router";
+            text = ''
+              PYTHONPATH="${packagePath}" ${routerUnwrapped}/bin/text-generation-router "$@"
+            '';
+          };
+        server = pkgs.python3.pkgs.callPackage ./nix/server.nix { inherit nix-filter; };
+        client = pkgs.python3.pkgs.callPackage ./nix/client.nix { };
+      in
+      {
+        checks = {
+          rust =
+            with pkgs;
+            rustPlatform.buildRustPackage {
+              name = "rust-checks";
+              src = ./.;
+              cargoLock = {
+                lockFile = ./Cargo.lock;
+              };
+              buildInputs = [ openssl.dev ];
+              nativeBuildInputs = [
+                clippy
+                pkg-config
+                protobuf
+                python3
+                rustfmt
+              ];
+              buildPhase = ''
+                cargo check
+              '';
+              checkPhase = ''
+                cargo fmt -- --check
+                cargo test -j $NIX_BUILD_CORES
+                cargo clippy
+              '';
+              installPhase = "touch $out";
+            };
+        };
+        formatter = pkgs.nixfmt-rfc-style;
+        devShells = with pkgs; rec {
+          default = pure;
+
+          pure = mkShell {
+            buildInputs = [
+              benchmark
+              launcher
+              router
+              server
+            ];
+          };
+          test = mkShell {
+            buildInputs =
+              [
+                benchmark
+                launcher
+                router
+                server
+                client
+                openssl.dev
+                pkg-config
+                cargo
+                rustfmt
+                clippy
+              ]
+              ++ (with python3.pkgs; [
+                docker
+                pytest
+                pytest-asyncio
+                syrupy
+                pre-commit
+                ruff
+              ]);
+          };
+
+          impure = callPackage ./nix/impure-shell.nix { inherit server; };
+
+          impure-flash-attn-v1 = callPackage ./nix/impure-shell.nix {
+            server = server.override { flash-attn = python3.pkgs.flash-attn-v1; };
+          };
+        };
+
+        packages = rec {
+          default = pkgs.writeShellApplication {
+            name = "text-generation-inference";
+            runtimeInputs = [
+              server
+              router
+            ];
+            text = ''
+              ${launcher}/bin/text-generation-launcher "$@"
+            '';
+          };
+
+          dockerImage = pkgs.callPackage nix/docker.nix {
+            text-generation-inference = default;
+          };
+
+          dockerImageStreamed = pkgs.callPackage nix/docker.nix {
+            text-generation-inference = default;
+            stream = true;
+          };
+        };
+      }
+    );
+}
diff --git a/integration-tests/conftest.py b/integration-tests/conftest.py
index 7665f589..14523cd4 100644
--- a/integration-tests/conftest.py
+++ b/integration-tests/conftest.py
@@ -1,47 +1,71 @@
-import sys
-import subprocess
-import contextlib
-import pytest
 import asyncio
-import os
-import docker
+import contextlib
 import json
 import math
-import time
+import os
 import random
-import re
+import shutil
+import subprocess
+import sys
+import tempfile
+import time
+from typing import Dict, List, Optional
 
-from docker.errors import NotFound
-from typing import Optional, List, Dict
-from syrupy.extensions.json import JSONSnapshotExtension
+import docker
+import pytest
 from aiohttp import ClientConnectorError, ClientOSError, ServerDisconnectedError
-
+from docker.errors import NotFound
+from syrupy.extensions.json import JSONSnapshotExtension
 from text_generation import AsyncClient
 from text_generation.types import (
-    Response,
-    Details,
-    InputToken,
-    Token,
     BestOfSequence,
-    Grammar,
+    Message,
     ChatComplete,
     ChatCompletionChunk,
     ChatCompletionComplete,
     Completion,
+    Details,
+    Grammar,
+    InputToken,
+    Response,
+    Token,
 )
 
 DOCKER_IMAGE = os.getenv("DOCKER_IMAGE", None)
 HUGGING_FACE_HUB_TOKEN = os.getenv("HF_TOKEN") or os.getenv("HUGGING_FACE_HUB_TOKEN")
 DOCKER_VOLUME = os.getenv("DOCKER_VOLUME", "/data")
+DOCKER_DEVICES = os.getenv("DOCKER_DEVICES")
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--release", action="store_true", default=False, help="run release tests"
+    )
+
+
+def pytest_configure(config):
+    config.addinivalue_line("markers", "release: mark test as a release-only test")
+
+
+def pytest_collection_modifyitems(config, items):
+    if config.getoption("--release"):
+        # --release given in cli: do not skip release tests
+        return
+    skip_release = pytest.mark.skip(reason="need --release option to run")
+    for item in items:
+        if "release" in item.keywords:
+            item.add_marker(skip_release)
 
 
 class ResponseComparator(JSONSnapshotExtension):
     rtol = 0.2
+    ignore_logprob = False
 
     def serialize(
         self,
         data,
         *,
+        include=None,
         exclude=None,
         matcher=None,
     ):
@@ -57,7 +81,12 @@ class ResponseComparator(JSONSnapshotExtension):
             data = [d.model_dump() for d in data]
 
         data = self._filter(
-            data=data, depth=0, path=(), exclude=exclude, matcher=matcher
+            data=data,
+            depth=0,
+            path=(),
+            exclude=exclude,
+            include=include,
+            matcher=matcher,
         )
         return json.dumps(data, indent=2, ensure_ascii=False, sort_keys=False) + "\n"
 
@@ -69,32 +98,36 @@ class ResponseComparator(JSONSnapshotExtension):
     ) -> bool:
         def convert_data(data):
             data = json.loads(data)
-            if isinstance(data, Dict) and "choices" in data:
-                choices = data["choices"]
-                if isinstance(choices, List) and len(choices) >= 1:
-                    if "delta" in choices[0]:
-                        return ChatCompletionChunk(**data)
-                    if "text" in choices[0]:
-                        return Completion(**data)
-                return ChatComplete(**data)
+            return _convert_data(data)
 
+        def _convert_data(data):
             if isinstance(data, Dict):
-                return Response(**data)
+                if "choices" in data:
+                    data["choices"] = list(
+                        sorted(data["choices"], key=lambda x: x["index"])
+                    )
+                    choices = data["choices"]
+                    if isinstance(choices, List) and len(choices) >= 1:
+                        if "delta" in choices[0]:
+                            return ChatCompletionChunk(**data)
+                        if "text" in choices[0]:
+                            return Completion(**data)
+                    return ChatComplete(**data)
+                else:
+                    return Response(**data)
             if isinstance(data, List):
-                if (
-                    len(data) > 0
-                    and "object" in data[0]
-                    and data[0]["object"] == "text_completion"
-                ):
-                    return [Completion(**d) for d in data]
-                return [Response(**d) for d in data]
+                return [_convert_data(d) for d in data]
             raise NotImplementedError
 
         def eq_token(token: Token, other: Token) -> bool:
             return (
                 token.id == other.id
                 and token.text == other.text
-                and math.isclose(token.logprob, other.logprob, rel_tol=self.rtol)
+                and (
+                    self.ignore_logprob
+                    or (token.logprob == other.logprob and token.logprob is None)
+                    or math.isclose(token.logprob, other.logprob, rel_tol=self.rtol)
+                )
                 and token.special == other.special
             )
 
@@ -104,8 +137,11 @@ class ResponseComparator(JSONSnapshotExtension):
                     prefill_token.id == other.id
                     and prefill_token.text == other.text
                     and (
-                        math.isclose(
-                            prefill_token.logprob, other.logprob, rel_tol=self.rtol
+                        self.ignore_logprob
+                        or math.isclose(
+                            prefill_token.logprob,
+                            other.logprob,
+                            rel_tol=self.rtol,
                         )
                         if prefill_token.logprob is not None
                         else prefill_token.logprob == other.logprob
@@ -222,9 +258,13 @@ class GenerousResponseComparator(ResponseComparator):
     rtol = 0.75
 
 
+class IgnoreLogProbResponseComparator(ResponseComparator):
+    ignore_logprob = True
+
+
 class LauncherHandle:
     def __init__(self, port: int):
-        self.client = AsyncClient(f"http://localhost:{port}")
+        self.client = AsyncClient(f"http://localhost:{port}", timeout=30)
 
     def _inner_health(self):
         raise NotImplementedError
@@ -238,7 +278,7 @@ class LauncherHandle:
             try:
                 await self.client.generate("test")
                 return
-            except (ClientConnectorError, ClientOSError, ServerDisconnectedError) as e:
+            except (ClientConnectorError, ClientOSError, ServerDisconnectedError):
                 time.sleep(1)
         raise RuntimeError("Health check failed")
 
@@ -273,6 +313,11 @@ def generous_response_snapshot(snapshot):
     return snapshot.use_extension(GenerousResponseComparator)
 
 
+@pytest.fixture
+def ignore_logprob_response_snapshot(snapshot):
+    return snapshot.use_extension(IgnoreLogProbResponseComparator)
+
+
 @pytest.fixture(scope="module")
 def event_loop():
     loop = asyncio.get_event_loop()
@@ -295,6 +340,9 @@ def launcher(event_loop):
         max_input_length: Optional[int] = None,
         max_batch_prefill_tokens: Optional[int] = None,
         max_total_tokens: Optional[int] = None,
+        lora_adapters: Optional[List[str]] = None,
+        cuda_graphs: Optional[List[int]] = None,
+        attention: Optional[str] = None,
     ):
         port = random.randint(8000, 10_000)
         master_port = random.randint(10_000, 20_000)
@@ -341,25 +389,38 @@ def launcher(event_loop):
         if max_total_tokens:
             args.append("--max-total-tokens")
             args.append(str(max_total_tokens))
+        if lora_adapters:
+            args.append("--lora-adapters")
+            args.append(",".join(lora_adapters))
+        if cuda_graphs:
+            args.append("--cuda-graphs")
+            args.append(",".join(map(str, cuda_graphs)))
+
+        print(" ".join(args), file=sys.stderr)
 
         env["LOG_LEVEL"] = "info,text_generation_router=debug"
 
         if not use_flash_attention:
             env["USE_FLASH_ATTENTION"] = "false"
+        if attention is not None:
+            env["ATTENTION"] = attention
 
-        with subprocess.Popen(
-            args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env
-        ) as process:
-            yield ProcessLauncherHandle(process, port)
+        with tempfile.TemporaryFile("w+") as tmp:
+            # We'll output stdout/stderr to a temporary file. Using a pipe
+            # cause the process to block until stdout is read.
+            with subprocess.Popen(
+                args,
+                stdout=tmp,
+                stderr=subprocess.STDOUT,
+                env=env,
+            ) as process:
+                yield ProcessLauncherHandle(process, port)
 
-            process.terminate()
-            process.wait(60)
+                process.terminate()
+                process.wait(60)
 
-            launcher_output = process.stdout.read().decode("utf-8")
-            print(launcher_output, file=sys.stderr)
-
-            process.stdout.close()
-            process.stderr.close()
+                tmp.seek(0)
+                shutil.copyfileobj(tmp, sys.stderr)
 
         if not use_flash_attention:
             del env["USE_FLASH_ATTENTION"]
@@ -377,6 +438,9 @@ def launcher(event_loop):
         max_input_length: Optional[int] = None,
         max_batch_prefill_tokens: Optional[int] = None,
         max_total_tokens: Optional[int] = None,
+        lora_adapters: Optional[List[str]] = None,
+        cuda_graphs: Optional[List[int]] = None,
+        attention: Optional[str] = None,
     ):
         port = random.randint(8000, 10_000)
 
@@ -406,6 +470,12 @@ def launcher(event_loop):
         if max_total_tokens:
             args.append("--max-total-tokens")
             args.append(str(max_total_tokens))
+        if lora_adapters:
+            args.append("--lora-adapters")
+            args.append(",".join(lora_adapters))
+        if cuda_graphs:
+            args.append("--cuda-graphs")
+            args.append(",".join(map(str, cuda_graphs)))
 
         client = docker.from_env()
 
@@ -425,6 +495,8 @@ def launcher(event_loop):
         }
         if not use_flash_attention:
             env["USE_FLASH_ATTENTION"] = "false"
+        if attention is not None:
+            env["ATTENTION"] = attention
 
         if HUGGING_FACE_HUB_TOKEN is not None:
             env["HF_TOKEN"] = HUGGING_FACE_HUB_TOKEN
@@ -433,6 +505,18 @@ def launcher(event_loop):
         if DOCKER_VOLUME:
             volumes = [f"{DOCKER_VOLUME}:/data"]
 
+        if DOCKER_DEVICES:
+            devices = DOCKER_DEVICES.split(",")
+            visible = os.getenv("ROCR_VISIBLE_DEVICES")
+            if visible:
+                env["ROCR_VISIBLE_DEVICES"] = visible
+            device_requests = []
+        else:
+            devices = []
+            device_requests = [
+                docker.types.DeviceRequest(count=gpu_count, capabilities=[["gpu"]])
+            ]
+
         container = client.containers.run(
             DOCKER_IMAGE,
             command=args,
@@ -440,11 +524,11 @@ def launcher(event_loop):
             environment=env,
             auto_remove=False,
             detach=True,
-            device_requests=[
-                docker.types.DeviceRequest(count=gpu_count, capabilities=[["gpu"]])
-            ],
+            device_requests=device_requests,
+            devices=devices,
             volumes=volumes,
             ports={"80/tcp": port},
+            healthcheck={"timeout": int(10 * 1e9)},
             shm_size="1G",
         )
 
@@ -495,3 +579,38 @@ def generate_load():
         return await asyncio.gather(*futures)
 
     return generate_load_inner
+
+
+@pytest.fixture(scope="module")
+def generate_multi():
+    async def generate_load_inner(
+        client: AsyncClient,
+        prompts: List[str],
+        max_new_tokens: int,
+        seed: Optional[int] = None,
+    ) -> List[Response]:
+
+        import numpy as np
+
+        arange = np.arange(len(prompts))
+        perm = np.random.permutation(arange)
+        rperm = [-1] * len(perm)
+        for i, p in enumerate(perm):
+            rperm[p] = i
+
+        shuffled_prompts = [prompts[p] for p in perm]
+        futures = [
+            client.chat(
+                messages=[Message(role="user", content=prompt)],
+                max_tokens=max_new_tokens,
+                temperature=0,
+                seed=seed,
+            )
+            for prompt in shuffled_prompts
+        ]
+
+        shuffled_responses = await asyncio.gather(*futures)
+        responses = [shuffled_responses[p] for p in rperm]
+        return responses
+
+    return generate_load_inner
diff --git a/integration-tests/models/__snapshots__/test_bloom_560m/test_bloom_560m.json b/integration-tests/models/__snapshots__/test_bloom_560m/test_bloom_560m.json
index 53a4ab85..54c66408 100644
--- a/integration-tests/models/__snapshots__/test_bloom_560m/test_bloom_560m.json
+++ b/integration-tests/models/__snapshots__/test_bloom_560m/test_bloom_560m.json
@@ -11,42 +11,42 @@
       },
       {
         "id": 49833,
-        "logprob": -10.5625,
+        "logprob": -10.5703125,
         "text": " dég"
       },
       {
         "id": 21543,
-        "logprob": -0.14770508,
+        "logprob": -0.14746094,
         "text": "uster"
       },
       {
         "id": 447,
-        "logprob": -1.9287109,
+        "logprob": -1.9277344,
         "text": " un"
       },
       {
         "id": 46341,
-        "logprob": -15.4609375,
+        "logprob": -15.421875,
         "text": " ort"
       },
       {
         "id": 35567,
-        "logprob": -7.5585938,
+        "logprob": -7.5820312,
         "text": "olan"
       },
       {
         "id": 15,
-        "logprob": -1.4003906,
+        "logprob": -1.4013672,
         "text": ","
       },
       {
         "id": 1669,
-        "logprob": -1.5673828,
+        "logprob": -1.5595703,
         "text": " il"
       },
       {
         "id": 11580,
-        "logprob": -0.94628906,
+        "logprob": -0.9428711,
         "text": " faut"
       },
       {
@@ -56,7 +56,7 @@
       },
       {
         "id": 39261,
-        "logprob": -1.5732422,
+        "logprob": -1.7763672,
         "text": " d'abord"
       }
     ],
@@ -64,65 +64,66 @@
     "tokens": [
       {
         "id": 578,
-        "logprob": -1.6591797,
+        "logprob": -1.7822266,
         "special": false,
         "text": " le"
       },
       {
         "id": 5608,
-        "logprob": -2.4492188,
+        "logprob": -2.4882812,
         "special": false,
         "text": " faire"
       },
       {
-        "id": 159570,
-        "logprob": -6.6835938,
+        "id": 7735,
+        "logprob": -2.4199219,
         "special": false,
-        "text": " réch"
+        "text": " fond"
       },
       {
-        "id": 810,
+        "id": 289,
         "logprob": 0.0,
         "special": false,
-        "text": "au"
+        "text": "re"
       },
       {
-        "id": 12736,
+        "id": 693,
+        "logprob": -2.4628906,
+        "special": false,
+        "text": " à"
+      },
+      {
+        "id": 366,
+        "logprob": -1.1308594,
+        "special": false,
+        "text": " la"
+      },
+      {
+        "id": 48844,
+        "logprob": -1.7900391,
+        "special": false,
+        "text": " cass"
+      },
+      {
+        "id": 1744,
         "logprob": 0.0,
         "special": false,
-        "text": "ffer"
+        "text": "ero"
       },
       {
-        "id": 1742,
-        "logprob": -2.5175781,
-        "special": false,
-        "text": " au"
-      },
-      {
-        "id": 6105,
-        "logprob": -2.0078125,
-        "special": false,
-        "text": " bain"
-      },
-      {
-        "id": 88254,
-        "logprob": -0.12695312,
-        "special": false,
-        "text": "-mar"
-      },
-      {
-        "id": 641,
+        "id": 327,
         "logprob": 0.0,
         "special": false,
-        "text": "ie"
+        "text": "le"
       },
       {
         "id": 2940,
-        "logprob": -3.5175781,
+        "logprob": -1.9306641,
         "special": false,
         "text": " avec"
       }
-    ]
+    ],
+    "top_tokens": null
   },
-  "generated_text": " le faire réchauffer au bain-marie avec"
+  "generated_text": " le faire fondre à la casserole avec"
 }
diff --git a/integration-tests/models/__snapshots__/test_bloom_560m/test_bloom_560m_all_params.json b/integration-tests/models/__snapshots__/test_bloom_560m/test_bloom_560m_all_params.json
index ace73416..9422f27f 100644
--- a/integration-tests/models/__snapshots__/test_bloom_560m/test_bloom_560m_all_params.json
+++ b/integration-tests/models/__snapshots__/test_bloom_560m/test_bloom_560m_all_params.json
@@ -11,7 +11,7 @@
       },
       {
         "id": 1669,
-        "logprob": -5.4414062,
+        "logprob": -5.4453125,
         "text": " il"
       },
       {
@@ -21,12 +21,12 @@
       },
       {
         "id": 3913,
-        "logprob": -4.3554688,
+        "logprob": -4.3320312,
         "text": " tout"
       },
       {
         "id": 39261,
-        "logprob": -2.9238281,
+        "logprob": -2.9160156,
         "text": " d'abord"
       }
     ],
@@ -34,65 +34,66 @@
     "tokens": [
       {
         "id": 408,
-        "logprob": -0.07891846,
+        "logprob": -0.16687012,
         "special": false,
         "text": " que"
       },
       {
         "id": 366,
-        "logprob": -1.2939453,
+        "logprob": -1.5517578,
         "special": false,
         "text": " la"
       },
       {
         "id": 8769,
-        "logprob": -0.3708496,
+        "logprob": -0.16687012,
         "special": false,
         "text": " personne"
       },
       {
         "id": 1479,
-        "logprob": -2.2871094,
+        "logprob": -2.1035156,
         "special": false,
         "text": " qui"
       },
       {
-        "id": 2997,
-        "logprob": -0.8671875,
+        "id": 143926,
+        "logprob": -2.8671875,
         "special": false,
-        "text": " vous"
+        "text": " réalise"
       },
       {
-        "id": 35977,
-        "logprob": -1.5097656,
+        "id": 578,
+        "logprob": 0.0,
         "special": false,
-        "text": " suit"
+        "text": " le"
       },
       {
-        "id": 21558,
-        "logprob": -0.07891846,
+        "id": 8138,
+        "logprob": -0.66748047,
         "special": false,
-        "text": " ait"
+        "text": " projet"
       },
       {
-        "id": 447,
-        "logprob": -0.12695312,
+        "id": 795,
+        "logprob": -1.6279297,
         "special": false,
-        "text": " un"
+        "text": " ne"
       },
       {
-        "id": 78606,
-        "logprob": -2.21875,
+        "id": 9802,
+        "logprob": -0.47875977,
         "special": false,
-        "text": " profil"
+        "text": " soit"
       },
       {
-        "id": 3899,
-        "logprob": -1.3535156,
+        "id": 1230,
+        "logprob": 0.0,
         "special": false,
-        "text": " bien"
+        "text": " pas"
       }
-    ]
+    ],
+    "top_tokens": null
   },
-  "generated_text": "Pour déguster un ortolan, il faut tout d'abord que la personne qui vous suit ait un profil bien"
+  "generated_text": "Pour déguster un ortolan, il faut tout d'abord que la personne qui réalise le projet ne soit pas"
 }
diff --git a/integration-tests/models/__snapshots__/test_bloom_560m_sharded/test_bloom_560m_sharded.json b/integration-tests/models/__snapshots__/test_bloom_560m_sharded/test_bloom_560m_sharded.json
index dd8936af..b17c889e 100644
--- a/integration-tests/models/__snapshots__/test_bloom_560m_sharded/test_bloom_560m_sharded.json
+++ b/integration-tests/models/__snapshots__/test_bloom_560m_sharded/test_bloom_560m_sharded.json
@@ -11,52 +11,52 @@
       },
       {
         "id": 49833,
-        "logprob": -10.5390625,
+        "logprob": -10.546875,
         "text": " dég"
       },
       {
         "id": 21543,
-        "logprob": -0.14758301,
+        "logprob": -0.14819336,
         "text": "uster"
       },
       {
         "id": 447,
-        "logprob": -1.9296875,
+        "logprob": -1.9257812,
         "text": " un"
       },
       {
         "id": 46341,
-        "logprob": -15.4453125,
+        "logprob": -15.4296875,
         "text": " ort"
       },
       {
         "id": 35567,
-        "logprob": -7.59375,
+        "logprob": -7.5625,
         "text": "olan"
       },
       {
         "id": 15,
-        "logprob": -1.3994141,
+        "logprob": -1.4199219,
         "text": ","
       },
       {
         "id": 1669,
-        "logprob": -1.578125,
+        "logprob": -1.5634766,
         "text": " il"
       },
       {
         "id": 11580,
-        "logprob": -0.9453125,
+        "logprob": -0.9458008,
         "text": " faut"
       },
       {
         "id": 3913,
-        "logprob": -3.7011719,
+        "logprob": -3.6816406,
         "text": " tout"
       },
       {
         "id": 39261,
-        "logprob": -1.5732422,
+        "logprob": -1.7753906,
         "text": " d'abord"
       }
     ],
@@ -64,65 +64,66 @@
     "tokens": [
       {
         "id": 578,
-        "logprob": -1.6474609,
+        "logprob": -1.828125,
         "special": false,
         "text": " le"
       },
       {
         "id": 5608,
-        "logprob": -2.5097656,
+        "logprob": -2.5546875,
         "special": false,
         "text": " faire"
       },
       {
-        "id": 159570,
-        "logprob": -6.65625,
+        "id": 7735,
+        "logprob": -2.4277344,
         "special": false,
-        "text": " réch"
+        "text": " fond"
       },
       {
-        "id": 810,
+        "id": 289,
         "logprob": 0.0,
         "special": false,
-        "text": "au"
+        "text": "re"
       },
       {
-        "id": 12736,
+        "id": 693,
+        "logprob": -2.4472656,
+        "special": false,
+        "text": " à"
+      },
+      {
+        "id": 366,
+        "logprob": -1.1494141,
+        "special": false,
+        "text": " la"
+      },
+      {
+        "id": 48844,
+        "logprob": -1.7939453,
+        "special": false,
+        "text": " cass"
+      },
+      {
+        "id": 1744,
         "logprob": 0.0,
         "special": false,
-        "text": "ffer"
+        "text": "ero"
       },
       {
-        "id": 1742,
-        "logprob": -2.5859375,
-        "special": false,
-        "text": " au"
-      },
-      {
-        "id": 6105,
-        "logprob": -2.03125,
-        "special": false,
-        "text": " bain"
-      },
-      {
-        "id": 88254,
-        "logprob": -0.12695312,
-        "special": false,
-        "text": "-mar"
-      },
-      {
-        "id": 641,
+        "id": 327,
         "logprob": 0.0,
         "special": false,
-        "text": "ie"
+        "text": "le"
       },
       {
         "id": 2940,
-        "logprob": -3.5175781,
+        "logprob": -1.9013672,
         "special": false,
         "text": " avec"
       }
-    ]
+    ],
+    "top_tokens": null
   },
-  "generated_text": " le faire réchauffer au bain-marie avec"
+  "generated_text": " le faire fondre à la casserole avec"
 }
diff --git a/integration-tests/models/__snapshots__/test_chat_llama/test_flash_llama_simple.json b/integration-tests/models/__snapshots__/test_chat_llama/test_flash_llama_simple.json
index 4cb548d2..5553e17d 100644
--- a/integration-tests/models/__snapshots__/test_chat_llama/test_flash_llama_simple.json
+++ b/integration-tests/models/__snapshots__/test_chat_llama/test_flash_llama_simple.json
@@ -5,7 +5,7 @@
       "index": 0,
       "logprobs": null,
       "message": {
-        "content": "As of today, there is a Update available for the Brooklyn, New York, area. According to the latest forecast, it's warm with high temperatures throughout the day. It's forecasted at 75°F for today and 77°F for tomorrow. However, in autumn, the weather typically changes drastically, becoming cooler and wetter. You can find the current weather forecast for the area through your local weather service. Additionally",
+        "content": "As of your last question, the weather in Brooklyn, New York, is typically hot and humid throughout the year. The suburbs around New York City are jealously sheltered, and at least in the Lower Bronx, there are very few outdoor environments to appreciate nature.\n\nIn terms of temperature, the warmest times of the year are from June to August, when average high temperatures typically range from around 73°F or 23°C",
         "name": null,
         "role": "assistant",
         "tool_calls": null
@@ -13,14 +13,14 @@
       "usage": null
     }
   ],
-  "created": 1712874856,
+  "created": 1724792495,
   "id": "",
   "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
-  "object": "text_completion",
-  "system_fingerprint": "2.0.1-native",
+  "object": "chat.completion",
+  "system_fingerprint": "2.2.1-dev0-native",
   "usage": {
     "completion_tokens": 100,
-    "prompt_tokens": 60,
-    "total_tokens": 160
+    "prompt_tokens": 61,
+    "total_tokens": 161
   }
 }
diff --git a/integration-tests/models/__snapshots__/test_completion_prompts/test_flash_llama_completion_many_prompts.json b/integration-tests/models/__snapshots__/test_completion_prompts/test_flash_llama_completion_many_prompts.json
index 99c33cf7..25b8120d 100644
--- a/integration-tests/models/__snapshots__/test_completion_prompts/test_flash_llama_completion_many_prompts.json
+++ b/integration-tests/models/__snapshots__/test_completion_prompts/test_flash_llama_completion_many_prompts.json
@@ -1,38 +1,38 @@
 {
   "choices": [
-    {
-      "finish_reason": "eos_token",
-      "index": 1,
-      "logprobs": null,
-      "text": " PR for more information?"
-    },
     {
       "finish_reason": "length",
       "index": 0,
       "logprobs": null,
-      "text": "le Business Incubator is providing a workspace"
+      "text": " A Beginner’s Guide\nDeep learning is a subset"
     },
     {
       "finish_reason": "length",
-      "index": 2,
+      "index": 1,
       "logprobs": null,
-      "text": " severely flawed and often has a substandard"
+      "text": " This is a question that has puzzled many people for"
     },
     {
       "finish_reason": "length",
       "index": 3,
       "logprobs": null,
-      "text": "hd20220811-"
+      "text": "usculas_minusculas(s):\n    \"\"\"\n"
+    },
+    {
+      "finish_reason": "length",
+      "index": 2,
+      "logprobs": null,
+      "text": " Paris\nWhat is the capital of France?\nThe"
     }
   ],
-  "created": 1713284455,
+  "created": 1725877154,
   "id": "",
-  "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
+  "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
   "object": "text_completion",
-  "system_fingerprint": "2.0.1-native",
+  "system_fingerprint": "2.2.1-dev0-native",
   "usage": {
-    "completion_tokens": 36,
-    "prompt_tokens": 8,
-    "total_tokens": 44
+    "completion_tokens": 40,
+    "prompt_tokens": 22,
+    "total_tokens": 62
   }
 }
diff --git a/integration-tests/models/__snapshots__/test_completion_prompts/test_flash_llama_completion_many_prompts_stream.json b/integration-tests/models/__snapshots__/test_completion_prompts/test_flash_llama_completion_many_prompts_stream.json
index d87071cf..dd22ceae 100644
--- a/integration-tests/models/__snapshots__/test_completion_prompts/test_flash_llama_completion_many_prompts_stream.json
+++ b/integration-tests/models/__snapshots__/test_completion_prompts/test_flash_llama_completion_many_prompts_stream.json
@@ -5,14 +5,14 @@
         "finish_reason": "",
         "index": 0,
         "logprobs": null,
-        "text": "\n"
+        "text": " A"
       }
     ],
-    "created": 1713284431,
+    "created": 1725883643,
     "id": "",
-    "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
@@ -20,14 +20,74 @@
         "finish_reason": "",
         "index": 1,
         "logprobs": null,
-        "text": "\n"
+        "text": " This"
       }
     ],
-    "created": 1713284431,
+    "created": 1725883643,
     "id": "",
-    "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "",
+        "index": 2,
+        "logprobs": null,
+        "text": " Paris"
+      }
+    ],
+    "created": 1725883643,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "text_completion",
+    "system_fingerprint": "2.2.1-dev0-native"
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "",
+        "index": 3,
+        "logprobs": null,
+        "text": "us"
+      }
+    ],
+    "created": 1725883643,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "text_completion",
+    "system_fingerprint": "2.2.1-dev0-native"
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "",
+        "index": 0,
+        "logprobs": null,
+        "text": " Beginner"
+      }
+    ],
+    "created": 1725883643,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "text_completion",
+    "system_fingerprint": "2.2.1-dev0-native"
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "",
+        "index": 1,
+        "logprobs": null,
+        "text": " is"
+      }
+    ],
+    "created": 1725883643,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "text_completion",
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
@@ -38,11 +98,11 @@
         "text": "\n"
       }
     ],
-    "created": 1713284431,
+    "created": 1725883643,
     "id": "",
-    "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
@@ -50,14 +110,14 @@
         "finish_reason": "",
         "index": 3,
         "logprobs": null,
-        "text": "hd"
+        "text": "cul"
       }
     ],
-    "created": 1713284431,
+    "created": 1725883643,
     "id": "",
-    "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
@@ -65,14 +125,14 @@
         "finish_reason": "",
         "index": 0,
         "logprobs": null,
-        "text": "\n"
+        "text": "’s"
       }
     ],
-    "created": 1713284431,
+    "created": 1725883643,
     "id": "",
-    "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
@@ -80,14 +140,14 @@
         "finish_reason": "",
         "index": 1,
         "logprobs": null,
-        "text": "\n"
+        "text": " a"
       }
     ],
-    "created": 1713284431,
+    "created": 1725883643,
     "id": "",
-    "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
@@ -95,14 +155,14 @@
         "finish_reason": "",
         "index": 2,
         "logprobs": null,
-        "text": "\n"
+        "text": "What"
       }
     ],
-    "created": 1713284431,
+    "created": 1725883643,
     "id": "",
-    "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
@@ -110,14 +170,14 @@
         "finish_reason": "",
         "index": 3,
         "logprobs": null,
-        "text": "aho"
+        "text": "as"
       }
     ],
-    "created": 1713284431,
+    "created": 1725883643,
     "id": "",
-    "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
@@ -125,14 +185,14 @@
         "finish_reason": "",
         "index": 0,
         "logprobs": null,
-        "text": "2"
+        "text": " Guide"
       }
     ],
-    "created": 1713284431,
+    "created": 1725883643,
     "id": "",
-    "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
@@ -140,254 +200,14 @@
         "finish_reason": "",
         "index": 1,
         "logprobs": null,
-        "text": "2"
+        "text": " question"
       }
     ],
-    "created": 1713284431,
+    "created": 1725883643,
     "id": "",
-    "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
-  },
-  {
-    "choices": [
-      {
-        "finish_reason": "",
-        "index": 2,
-        "logprobs": null,
-        "text": "2"
-      }
-    ],
-    "created": 1713284431,
-    "id": "",
-    "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
-    "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
-  },
-  {
-    "choices": [
-      {
-        "finish_reason": "",
-        "index": 3,
-        "logprobs": null,
-        "text": "ima"
-      }
-    ],
-    "created": 1713284431,
-    "id": "",
-    "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
-    "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
-  },
-  {
-    "choices": [
-      {
-        "finish_reason": "",
-        "index": 0,
-        "logprobs": null,
-        "text": "."
-      }
-    ],
-    "created": 1713284431,
-    "id": "",
-    "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
-    "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
-  },
-  {
-    "choices": [
-      {
-        "finish_reason": "",
-        "index": 1,
-        "logprobs": null,
-        "text": "."
-      }
-    ],
-    "created": 1713284431,
-    "id": "",
-    "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
-    "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
-  },
-  {
-    "choices": [
-      {
-        "finish_reason": "",
-        "index": 2,
-        "logprobs": null,
-        "text": "."
-      }
-    ],
-    "created": 1713284431,
-    "id": "",
-    "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
-    "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
-  },
-  {
-    "choices": [
-      {
-        "finish_reason": "",
-        "index": 3,
-        "logprobs": null,
-        "text": "\n"
-      }
-    ],
-    "created": 1713284431,
-    "id": "",
-    "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
-    "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
-  },
-  {
-    "choices": [
-      {
-        "finish_reason": "",
-        "index": 0,
-        "logprobs": null,
-        "text": " Sarah"
-      }
-    ],
-    "created": 1713284431,
-    "id": "",
-    "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
-    "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
-  },
-  {
-    "choices": [
-      {
-        "finish_reason": "",
-        "index": 1,
-        "logprobs": null,
-        "text": " Yes"
-      }
-    ],
-    "created": 1713284431,
-    "id": "",
-    "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
-    "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
-  },
-  {
-    "choices": [
-      {
-        "finish_reason": "",
-        "index": 2,
-        "logprobs": null,
-        "text": " And"
-      }
-    ],
-    "created": 1713284431,
-    "id": "",
-    "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
-    "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
-  },
-  {
-    "choices": [
-      {
-        "finish_reason": "",
-        "index": 3,
-        "logprobs": null,
-        "text": "i"
-      }
-    ],
-    "created": 1713284431,
-    "id": "",
-    "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
-    "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
-  },
-  {
-    "choices": [
-      {
-        "finish_reason": "",
-        "index": 0,
-        "logprobs": null,
-        "text": "'"
-      }
-    ],
-    "created": 1713284431,
-    "id": "",
-    "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
-    "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
-  },
-  {
-    "choices": [
-      {
-        "finish_reason": "",
-        "index": 1,
-        "logprobs": null,
-        "text": ","
-      }
-    ],
-    "created": 1713284431,
-    "id": "",
-    "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
-    "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
-  },
-  {
-    "choices": [
-      {
-        "finish_reason": "",
-        "index": 2,
-        "logprobs": null,
-        "text": " what"
-      }
-    ],
-    "created": 1713284431,
-    "id": "",
-    "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
-    "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
-  },
-  {
-    "choices": [
-      {
-        "finish_reason": "",
-        "index": 3,
-        "logprobs": null,
-        "text": "'"
-      }
-    ],
-    "created": 1713284431,
-    "id": "",
-    "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
-    "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
-  },
-  {
-    "choices": [
-      {
-        "finish_reason": "",
-        "index": 0,
-        "logprobs": null,
-        "text": "s"
-      }
-    ],
-    "created": 1713284431,
-    "id": "",
-    "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
-    "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
-  },
-  {
-    "choices": [
-      {
-        "finish_reason": "",
-        "index": 1,
-        "logprobs": null,
-        "text": " Moh"
-      }
-    ],
-    "created": 1713284431,
-    "id": "",
-    "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
-    "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
@@ -398,11 +218,11 @@
         "text": " is"
       }
     ],
-    "created": 1713284431,
+    "created": 1725883643,
     "id": "",
-    "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
@@ -410,14 +230,14 @@
         "finish_reason": "",
         "index": 3,
         "logprobs": null,
-        "text": "m"
+        "text": "_minus"
       }
     ],
-    "created": 1713284431,
+    "created": 1725883643,
     "id": "",
-    "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
@@ -425,14 +245,14 @@
         "finish_reason": "",
         "index": 0,
         "logprobs": null,
-        "text": " Room"
+        "text": "\n"
       }
     ],
-    "created": 1713284431,
+    "created": 1725883643,
     "id": "",
-    "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
@@ -440,14 +260,14 @@
         "finish_reason": "",
         "index": 1,
         "logprobs": null,
-        "text": "s"
+        "text": " that"
       }
     ],
-    "created": 1713284431,
+    "created": 1725883643,
     "id": "",
-    "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
@@ -458,11 +278,11 @@
         "text": " the"
       }
     ],
-    "created": 1713284431,
+    "created": 1725883643,
     "id": "",
-    "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
@@ -470,14 +290,14 @@
         "finish_reason": "",
         "index": 3,
         "logprobs": null,
-        "text": " tired"
+        "text": "cul"
       }
     ],
-    "created": 1713284431,
+    "created": 1725883643,
     "id": "",
-    "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
@@ -485,14 +305,14 @@
         "finish_reason": "",
         "index": 0,
         "logprobs": null,
-        "text": ":"
+        "text": "Deep"
       }
     ],
-    "created": 1713284431,
+    "created": 1725883643,
     "id": "",
-    "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
@@ -500,14 +320,14 @@
         "finish_reason": "",
         "index": 1,
         "logprobs": null,
-        "text": "'"
+        "text": " has"
       }
     ],
-    "created": 1713284431,
+    "created": 1725883643,
     "id": "",
-    "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
@@ -518,11 +338,11 @@
         "text": " capital"
       }
     ],
-    "created": 1713284431,
+    "created": 1725883643,
     "id": "",
-    "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
@@ -530,14 +350,14 @@
         "finish_reason": "",
         "index": 3,
         "logprobs": null,
-        "text": " of"
+        "text": "as"
       }
     ],
-    "created": 1713284431,
+    "created": 1725883643,
     "id": "",
-    "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
@@ -545,14 +365,14 @@
         "finish_reason": "",
         "index": 0,
         "logprobs": null,
-        "text": " She"
+        "text": " learning"
       }
     ],
-    "created": 1713284431,
+    "created": 1725883643,
     "id": "",
-    "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
@@ -560,14 +380,14 @@
         "finish_reason": "",
         "index": 1,
         "logprobs": null,
-        "text": " scale"
+        "text": " puzzled"
       }
     ],
-    "created": 1713284431,
+    "created": 1725883643,
     "id": "",
-    "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
@@ -578,11 +398,11 @@
         "text": " of"
       }
     ],
-    "created": 1713284431,
+    "created": 1725883643,
     "id": "",
-    "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
@@ -590,13 +410,193 @@
         "finish_reason": "",
         "index": 3,
         "logprobs": null,
-        "text": " being"
+        "text": "(s"
       }
     ],
-    "created": 1713284431,
+    "created": 1725883643,
     "id": "",
-    "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "",
+        "index": 0,
+        "logprobs": null,
+        "text": " is"
+      }
+    ],
+    "created": 1725883643,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "text_completion",
+    "system_fingerprint": "2.2.1-dev0-native"
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "",
+        "index": 1,
+        "logprobs": null,
+        "text": " many"
+      }
+    ],
+    "created": 1725883643,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "text_completion",
+    "system_fingerprint": "2.2.1-dev0-native"
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "",
+        "index": 2,
+        "logprobs": null,
+        "text": " France"
+      }
+    ],
+    "created": 1725883643,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "text_completion",
+    "system_fingerprint": "2.2.1-dev0-native"
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "",
+        "index": 3,
+        "logprobs": null,
+        "text": "):\n"
+      }
+    ],
+    "created": 1725883643,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "text_completion",
+    "system_fingerprint": "2.2.1-dev0-native"
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "",
+        "index": 0,
+        "logprobs": null,
+        "text": " a"
+      }
+    ],
+    "created": 1725883643,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "text_completion",
+    "system_fingerprint": "2.2.1-dev0-native"
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "",
+        "index": 1,
+        "logprobs": null,
+        "text": " people"
+      }
+    ],
+    "created": 1725883643,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "text_completion",
+    "system_fingerprint": "2.2.1-dev0-native"
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "",
+        "index": 2,
+        "logprobs": null,
+        "text": "?\n"
+      }
+    ],
+    "created": 1725883643,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "text_completion",
+    "system_fingerprint": "2.2.1-dev0-native"
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "",
+        "index": 3,
+        "logprobs": null,
+        "text": "   "
+      }
+    ],
+    "created": 1725883643,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "text_completion",
+    "system_fingerprint": "2.2.1-dev0-native"
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "text": " subset"
+      }
+    ],
+    "created": 1725883643,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "text_completion",
+    "system_fingerprint": "2.2.1-dev0-native"
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 1,
+        "logprobs": null,
+        "text": " for"
+      }
+    ],
+    "created": 1725883643,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "text_completion",
+    "system_fingerprint": "2.2.1-dev0-native"
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 2,
+        "logprobs": null,
+        "text": "The"
+      }
+    ],
+    "created": 1725883643,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "text_completion",
+    "system_fingerprint": "2.2.1-dev0-native"
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 3,
+        "logprobs": null,
+        "text": " \"\"\"\n"
+      }
+    ],
+    "created": 1725883643,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "text_completion",
+    "system_fingerprint": "2.2.1-dev0-native"
   }
 ]
diff --git a/integration-tests/models/__snapshots__/test_completion_prompts/test_flash_llama_completion_single_prompt.json b/integration-tests/models/__snapshots__/test_completion_prompts/test_flash_llama_completion_single_prompt.json
index 5aed4935..7ad56271 100644
--- a/integration-tests/models/__snapshots__/test_completion_prompts/test_flash_llama_completion_single_prompt.json
+++ b/integration-tests/models/__snapshots__/test_completion_prompts/test_flash_llama_completion_single_prompt.json
@@ -4,17 +4,17 @@
       "finish_reason": "length",
       "index": 0,
       "logprobs": null,
-      "text": " PR for flake8"
+      "text": " A Beginner’s Guide\nDeep learning is a subset"
     }
   ],
-  "created": 1713284454,
+  "created": 1725876621,
   "id": "",
-  "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
+  "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
   "object": "text_completion",
-  "system_fingerprint": "2.0.1-native",
+  "system_fingerprint": "2.2.1-dev0-native",
   "usage": {
-    "completion_tokens": 5,
+    "completion_tokens": 10,
     "prompt_tokens": 6,
-    "total_tokens": 11
+    "total_tokens": 16
   }
 }
diff --git a/integration-tests/models/__snapshots__/test_completion_prompts/test_flash_llama_completion_stream_usage.json b/integration-tests/models/__snapshots__/test_completion_prompts/test_flash_llama_completion_stream_usage.json
new file mode 100644
index 00000000..8c7be4cb
--- /dev/null
+++ b/integration-tests/models/__snapshots__/test_completion_prompts/test_flash_llama_completion_stream_usage.json
@@ -0,0 +1,206 @@
+[
+  {
+    "choices": [
+      {
+        "delta": {
+          "content": "**",
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "finish_reason": null,
+        "index": 0,
+        "logprobs": null
+      }
+    ],
+    "created": 1726656043,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion.chunk",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": null
+  },
+  {
+    "choices": [
+      {
+        "delta": {
+          "content": "Deep",
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "finish_reason": null,
+        "index": 0,
+        "logprobs": null
+      }
+    ],
+    "created": 1726656043,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion.chunk",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": null
+  },
+  {
+    "choices": [
+      {
+        "delta": {
+          "content": " Learning",
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "finish_reason": null,
+        "index": 0,
+        "logprobs": null
+      }
+    ],
+    "created": 1726656043,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion.chunk",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": null
+  },
+  {
+    "choices": [
+      {
+        "delta": {
+          "content": ":",
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "finish_reason": null,
+        "index": 0,
+        "logprobs": null
+      }
+    ],
+    "created": 1726656043,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion.chunk",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": null
+  },
+  {
+    "choices": [
+      {
+        "delta": {
+          "content": " An",
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "finish_reason": null,
+        "index": 0,
+        "logprobs": null
+      }
+    ],
+    "created": 1726656043,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion.chunk",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": null
+  },
+  {
+    "choices": [
+      {
+        "delta": {
+          "content": " Overview",
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "finish_reason": null,
+        "index": 0,
+        "logprobs": null
+      }
+    ],
+    "created": 1726656043,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion.chunk",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": null
+  },
+  {
+    "choices": [
+      {
+        "delta": {
+          "content": "**\n",
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "finish_reason": null,
+        "index": 0,
+        "logprobs": null
+      }
+    ],
+    "created": 1726656044,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion.chunk",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": null
+  },
+  {
+    "choices": [
+      {
+        "delta": {
+          "content": "================================",
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "finish_reason": null,
+        "index": 0,
+        "logprobs": null
+      }
+    ],
+    "created": 1726656044,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion.chunk",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": null
+  },
+  {
+    "choices": [
+      {
+        "delta": {
+          "content": "=====",
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "finish_reason": null,
+        "index": 0,
+        "logprobs": null
+      }
+    ],
+    "created": 1726656044,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion.chunk",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": null
+  },
+  {
+    "choices": [
+      {
+        "delta": {
+          "content": "\n\n",
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null
+      }
+    ],
+    "created": 1726656044,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion.chunk",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 40,
+      "total_tokens": 50
+    }
+  }
+]
diff --git a/integration-tests/models/__snapshots__/test_flash_deepseek_v2/test_flash_deepseek_v2.json b/integration-tests/models/__snapshots__/test_flash_deepseek_v2/test_flash_deepseek_v2.json
new file mode 100644
index 00000000..732b0c49
--- /dev/null
+++ b/integration-tests/models/__snapshots__/test_flash_deepseek_v2/test_flash_deepseek_v2.json
@@ -0,0 +1,89 @@
+{
+  "details": {
+    "best_of_sequences": null,
+    "finish_reason": "length",
+    "generated_tokens": 10,
+    "prefill": [
+      {
+        "id": 100000,
+        "logprob": null,
+        "text": "<｜begin▁of▁sentence｜>"
+      },
+      {
+        "id": 3533,
+        "logprob": -9.625,
+        "text": "Test"
+      },
+      {
+        "id": 3102,
+        "logprob": -11.25,
+        "text": " request"
+      }
+    ],
+    "seed": null,
+    "tokens": [
+      {
+        "id": 185,
+        "logprob": -1.546875,
+        "special": false,
+        "text": "\n"
+      },
+      {
+        "id": 549,
+        "logprob": -2.859375,
+        "special": false,
+        "text": "The"
+      },
+      {
+        "id": 1727,
+        "logprob": -2.484375,
+        "special": false,
+        "text": " test"
+      },
+      {
+        "id": 3102,
+        "logprob": -0.83203125,
+        "special": false,
+        "text": " request"
+      },
+      {
+        "id": 317,
+        "logprob": -1.1484375,
+        "special": false,
+        "text": " is"
+      },
+      {
+        "id": 245,
+        "logprob": -1.578125,
+        "special": false,
+        "text": " a"
+      },
+      {
+        "id": 3412,
+        "logprob": -2.578125,
+        "special": false,
+        "text": " document"
+      },
+      {
+        "id": 344,
+        "logprob": -1.125,
+        "special": false,
+        "text": " that"
+      },
+      {
+        "id": 317,
+        "logprob": -1.6953125,
+        "special": false,
+        "text": " is"
+      },
+      {
+        "id": 1222,
+        "logprob": -1.71875,
+        "special": false,
+        "text": " used"
+      }
+    ],
+    "top_tokens": null
+  },
+  "generated_text": "\nThe test request is a document that is used"
+}
diff --git a/integration-tests/models/__snapshots__/test_flash_deepseek_v2/test_flash_deepseek_v2_all_params.json b/integration-tests/models/__snapshots__/test_flash_deepseek_v2/test_flash_deepseek_v2_all_params.json
new file mode 100644
index 00000000..6b45cf6b
--- /dev/null
+++ b/integration-tests/models/__snapshots__/test_flash_deepseek_v2/test_flash_deepseek_v2_all_params.json
@@ -0,0 +1,53 @@
+{
+  "details": {
+    "best_of_sequences": null,
+    "finish_reason": "eos_token",
+    "generated_tokens": 4,
+    "prefill": [
+      {
+        "id": 100000,
+        "logprob": null,
+        "text": "<｜begin▁of▁sentence｜>"
+      },
+      {
+        "id": 3533,
+        "logprob": -9.625,
+        "text": "Test"
+      },
+      {
+        "id": 3102,
+        "logprob": -11.25,
+        "text": " request"
+      }
+    ],
+    "seed": 0,
+    "tokens": [
+      {
+        "id": 2143,
+        "logprob": -1.828125,
+        "special": false,
+        "text": " sent"
+      },
+      {
+        "id": 10081,
+        "logprob": -0.41210938,
+        "special": false,
+        "text": " successfully"
+      },
+      {
+        "id": 13,
+        "logprob": 0.0,
+        "special": false,
+        "text": "."
+      },
+      {
+        "id": 100001,
+        "logprob": -0.16015625,
+        "special": true,
+        "text": "<｜end▁of▁sentence｜>"
+      }
+    ],
+    "top_tokens": null
+  },
+  "generated_text": "Test request sent successfully."
+}
diff --git a/integration-tests/models/__snapshots__/test_flash_deepseek_v2/test_flash_deepseek_v2_load.json b/integration-tests/models/__snapshots__/test_flash_deepseek_v2/test_flash_deepseek_v2_load.json
new file mode 100644
index 00000000..f1eeab25
--- /dev/null
+++ b/integration-tests/models/__snapshots__/test_flash_deepseek_v2/test_flash_deepseek_v2_load.json
@@ -0,0 +1,358 @@
+[
+  {
+    "details": {
+      "best_of_sequences": null,
+      "finish_reason": "length",
+      "generated_tokens": 10,
+      "prefill": [
+        {
+          "id": 100000,
+          "logprob": null,
+          "text": "<｜begin▁of▁sentence｜>"
+        },
+        {
+          "id": 3533,
+          "logprob": -9.625,
+          "text": "Test"
+        },
+        {
+          "id": 3102,
+          "logprob": -11.25,
+          "text": " request"
+        }
+      ],
+      "seed": null,
+      "tokens": [
+        {
+          "id": 185,
+          "logprob": -1.546875,
+          "special": false,
+          "text": "\n"
+        },
+        {
+          "id": 549,
+          "logprob": -2.859375,
+          "special": false,
+          "text": "The"
+        },
+        {
+          "id": 1727,
+          "logprob": -2.4375,
+          "special": false,
+          "text": " test"
+        },
+        {
+          "id": 3102,
+          "logprob": -0.83984375,
+          "special": false,
+          "text": " request"
+        },
+        {
+          "id": 317,
+          "logprob": -1.1328125,
+          "special": false,
+          "text": " is"
+        },
+        {
+          "id": 254,
+          "logprob": -1.515625,
+          "special": false,
+          "text": " the"
+        },
+        {
+          "id": 1022,
+          "logprob": -1.15625,
+          "special": false,
+          "text": " first"
+        },
+        {
+          "id": 3458,
+          "logprob": -0.3671875,
+          "special": false,
+          "text": " step"
+        },
+        {
+          "id": 279,
+          "logprob": -0.88671875,
+          "special": false,
+          "text": " in"
+        },
+        {
+          "id": 254,
+          "logprob": -0.69140625,
+          "special": false,
+          "text": " the"
+        }
+      ],
+      "top_tokens": null
+    },
+    "generated_text": "\nThe test request is the first step in the"
+  },
+  {
+    "details": {
+      "best_of_sequences": null,
+      "finish_reason": "length",
+      "generated_tokens": 10,
+      "prefill": [
+        {
+          "id": 100000,
+          "logprob": null,
+          "text": "<｜begin▁of▁sentence｜>"
+        },
+        {
+          "id": 3533,
+          "logprob": -9.625,
+          "text": "Test"
+        },
+        {
+          "id": 3102,
+          "logprob": -11.25,
+          "text": " request"
+        }
+      ],
+      "seed": null,
+      "tokens": [
+        {
+          "id": 185,
+          "logprob": -1.546875,
+          "special": false,
+          "text": "\n"
+        },
+        {
+          "id": 549,
+          "logprob": -2.859375,
+          "special": false,
+          "text": "The"
+        },
+        {
+          "id": 1727,
+          "logprob": -2.4375,
+          "special": false,
+          "text": " test"
+        },
+        {
+          "id": 3102,
+          "logprob": -0.83984375,
+          "special": false,
+          "text": " request"
+        },
+        {
+          "id": 317,
+          "logprob": -1.1328125,
+          "special": false,
+          "text": " is"
+        },
+        {
+          "id": 254,
+          "logprob": -1.515625,
+          "special": false,
+          "text": " the"
+        },
+        {
+          "id": 1022,
+          "logprob": -1.15625,
+          "special": false,
+          "text": " first"
+        },
+        {
+          "id": 3458,
+          "logprob": -0.3671875,
+          "special": false,
+          "text": " step"
+        },
+        {
+          "id": 279,
+          "logprob": -0.88671875,
+          "special": false,
+          "text": " in"
+        },
+        {
+          "id": 254,
+          "logprob": -0.69140625,
+          "special": false,
+          "text": " the"
+        }
+      ],
+      "top_tokens": null
+    },
+    "generated_text": "\nThe test request is the first step in the"
+  },
+  {
+    "details": {
+      "best_of_sequences": null,
+      "finish_reason": "length",
+      "generated_tokens": 10,
+      "prefill": [
+        {
+          "id": 100000,
+          "logprob": null,
+          "text": "<｜begin▁of▁sentence｜>"
+        },
+        {
+          "id": 3533,
+          "logprob": -9.625,
+          "text": "Test"
+        },
+        {
+          "id": 3102,
+          "logprob": -11.25,
+          "text": " request"
+        }
+      ],
+      "seed": null,
+      "tokens": [
+        {
+          "id": 185,
+          "logprob": -1.546875,
+          "special": false,
+          "text": "\n"
+        },
+        {
+          "id": 549,
+          "logprob": -2.859375,
+          "special": false,
+          "text": "The"
+        },
+        {
+          "id": 1727,
+          "logprob": -2.4375,
+          "special": false,
+          "text": " test"
+        },
+        {
+          "id": 3102,
+          "logprob": -0.83984375,
+          "special": false,
+          "text": " request"
+        },
+        {
+          "id": 317,
+          "logprob": -1.1328125,
+          "special": false,
+          "text": " is"
+        },
+        {
+          "id": 254,
+          "logprob": -1.515625,
+          "special": false,
+          "text": " the"
+        },
+        {
+          "id": 1022,
+          "logprob": -1.15625,
+          "special": false,
+          "text": " first"
+        },
+        {
+          "id": 3458,
+          "logprob": -0.3671875,
+          "special": false,
+          "text": " step"
+        },
+        {
+          "id": 279,
+          "logprob": -0.88671875,
+          "special": false,
+          "text": " in"
+        },
+        {
+          "id": 254,
+          "logprob": -0.69140625,
+          "special": false,
+          "text": " the"
+        }
+      ],
+      "top_tokens": null
+    },
+    "generated_text": "\nThe test request is the first step in the"
+  },
+  {
+    "details": {
+      "best_of_sequences": null,
+      "finish_reason": "length",
+      "generated_tokens": 10,
+      "prefill": [
+        {
+          "id": 100000,
+          "logprob": null,
+          "text": "<｜begin▁of▁sentence｜>"
+        },
+        {
+          "id": 3533,
+          "logprob": -9.625,
+          "text": "Test"
+        },
+        {
+          "id": 3102,
+          "logprob": -11.25,
+          "text": " request"
+        }
+      ],
+      "seed": null,
+      "tokens": [
+        {
+          "id": 185,
+          "logprob": -1.546875,
+          "special": false,
+          "text": "\n"
+        },
+        {
+          "id": 549,
+          "logprob": -2.859375,
+          "special": false,
+          "text": "The"
+        },
+        {
+          "id": 1727,
+          "logprob": -2.4375,
+          "special": false,
+          "text": " test"
+        },
+        {
+          "id": 3102,
+          "logprob": -0.83984375,
+          "special": false,
+          "text": " request"
+        },
+        {
+          "id": 317,
+          "logprob": -1.1328125,
+          "special": false,
+          "text": " is"
+        },
+        {
+          "id": 254,
+          "logprob": -1.515625,
+          "special": false,
+          "text": " the"
+        },
+        {
+          "id": 1022,
+          "logprob": -1.15625,
+          "special": false,
+          "text": " first"
+        },
+        {
+          "id": 3458,
+          "logprob": -0.3671875,
+          "special": false,
+          "text": " step"
+        },
+        {
+          "id": 279,
+          "logprob": -0.88671875,
+          "special": false,
+          "text": " in"
+        },
+        {
+          "id": 254,
+          "logprob": -0.69140625,
+          "special": false,
+          "text": " the"
+        }
+      ],
+      "top_tokens": null
+    },
+    "generated_text": "\nThe test request is the first step in the"
+  }
+]
diff --git a/integration-tests/models/__snapshots__/test_flash_gemma/test_flash_gemma.json b/integration-tests/models/__snapshots__/test_flash_gemma/test_flash_gemma.json
index 80f0d053..e322446b 100644
--- a/integration-tests/models/__snapshots__/test_flash_gemma/test_flash_gemma.json
+++ b/integration-tests/models/__snapshots__/test_flash_gemma/test_flash_gemma.json
@@ -11,12 +11,12 @@
       },
       {
         "id": 2015,
-        "logprob": -10.0,
+        "logprob": -10.0625,
         "text": "Test"
       },
       {
         "id": 3853,
-        "logprob": -10.875,
+        "logprob": -11.0,
         "text": " request"
       }
     ],
@@ -24,13 +24,13 @@
     "tokens": [
       {
         "id": 1736,
-        "logprob": -2.09375,
+        "logprob": -2.109375,
         "special": false,
         "text": " form"
       },
       {
         "id": 109,
-        "logprob": -1.8671875,
+        "logprob": -1.90625,
         "special": false,
         "text": "\n\n"
       },
@@ -42,43 +42,43 @@
       },
       {
         "id": 2121,
-        "logprob": -1.8203125,
+        "logprob": -1.796875,
         "special": false,
         "text": " test"
       },
       {
         "id": 3853,
-        "logprob": -0.23242188,
+        "logprob": -0.24511719,
         "special": false,
         "text": " request"
       },
       {
         "id": 1736,
-        "logprob": -0.08544922,
+        "logprob": -0.09326172,
         "special": false,
         "text": " form"
       },
       {
         "id": 603,
-        "logprob": -0.9375,
+        "logprob": -0.95703125,
         "special": false,
         "text": " is"
       },
       {
         "id": 1671,
-        "logprob": -1.671875,
+        "logprob": -1.5859375,
         "special": false,
         "text": " used"
       },
       {
         "id": 577,
-        "logprob": -0.40429688,
+        "logprob": -0.39257812,
         "special": false,
         "text": " to"
       },
       {
         "id": 3853,
-        "logprob": -1.1875,
+        "logprob": -1.25,
         "special": false,
         "text": " request"
       }
diff --git a/integration-tests/models/__snapshots__/test_flash_gemma/test_flash_gemma_all_params.json b/integration-tests/models/__snapshots__/test_flash_gemma/test_flash_gemma_all_params.json
index 8253dc96..0b840bfd 100644
--- a/integration-tests/models/__snapshots__/test_flash_gemma/test_flash_gemma_all_params.json
+++ b/integration-tests/models/__snapshots__/test_flash_gemma/test_flash_gemma_all_params.json
@@ -11,12 +11,12 @@
       },
       {
         "id": 2015,
-        "logprob": -10.0,
+        "logprob": -10.0625,
         "text": "Test"
       },
       {
         "id": 3853,
-        "logprob": -10.875,
+        "logprob": -11.0,
         "text": " request"
       }
     ],
@@ -24,7 +24,7 @@
     "tokens": [
       {
         "id": 7539,
-        "logprob": -0.73046875,
+        "logprob": -0.609375,
         "special": false,
         "text": " forms"
       },
@@ -36,7 +36,7 @@
       },
       {
         "id": 671,
-        "logprob": -1.703125,
+        "logprob": -1.5546875,
         "special": false,
         "text": " an"
       },
@@ -66,24 +66,24 @@
       },
       {
         "id": 11859,
-        "logprob": -1.6953125,
+        "logprob": -1.953125,
         "special": false,
         "text": " lab"
       },
       {
         "id": 2185,
-        "logprob": -1.3125,
+        "logprob": -1.7734375,
         "special": false,
         "text": " process"
       },
       {
-        "id": 578,
-        "logprob": -1.5,
+        "id": 235265,
+        "logprob": 0.0,
         "special": false,
-        "text": " and"
+        "text": "."
       }
     ],
     "top_tokens": null
   },
-  "generated_text": "Test request forms are an essential part of the lab process and"
+  "generated_text": "Test request forms are an essential part of the lab process."
 }
diff --git a/integration-tests/models/__snapshots__/test_flash_gemma2/test_flash_gemma2.json b/integration-tests/models/__snapshots__/test_flash_gemma2/test_flash_gemma2.json
new file mode 100644
index 00000000..1e9a50cf
--- /dev/null
+++ b/integration-tests/models/__snapshots__/test_flash_gemma2/test_flash_gemma2.json
@@ -0,0 +1,254 @@
+{
+  "details": {
+    "best_of_sequences": null,
+    "finish_reason": "length",
+    "generated_tokens": 10,
+    "prefill": [
+      {
+        "id": 2,
+        "logprob": null,
+        "text": "<bos>"
+      },
+      {
+        "id": 106,
+        "logprob": -47.25,
+        "text": "<start_of_turn>"
+      },
+      {
+        "id": 1645,
+        "logprob": -18.875,
+        "text": "user"
+      },
+      {
+        "id": 235292,
+        "logprob": -7.15625,
+        "text": ":"
+      },
+      {
+        "id": 108,
+        "logprob": -4.78125,
+        "text": "\n"
+      },
+      {
+        "id": 5559,
+        "logprob": -10.0,
+        "text": "Write"
+      },
+      {
+        "id": 476,
+        "logprob": -0.1171875,
+        "text": " a"
+      },
+      {
+        "id": 19592,
+        "logprob": -2.46875,
+        "text": " poem"
+      },
+      {
+        "id": 577,
+        "logprob": -5.84375,
+        "text": " to"
+      },
+      {
+        "id": 1707,
+        "logprob": -6.375,
+        "text": " help"
+      },
+      {
+        "id": 682,
+        "logprob": -2.125,
+        "text": " me"
+      },
+      {
+        "id": 5434,
+        "logprob": -1.546875,
+        "text": " remember"
+      },
+      {
+        "id": 573,
+        "logprob": -0.62890625,
+        "text": " the"
+      },
+      {
+        "id": 1370,
+        "logprob": -6.65625,
+        "text": " first"
+      },
+      {
+        "id": 235248,
+        "logprob": -1.84375,
+        "text": " "
+      },
+      {
+        "id": 235274,
+        "logprob": -0.45117188,
+        "text": "1"
+      },
+      {
+        "id": 235276,
+        "logprob": -0.07421875,
+        "text": "0"
+      },
+      {
+        "id": 6635,
+        "logprob": -2.109375,
+        "text": " elements"
+      },
+      {
+        "id": 611,
+        "logprob": -0.4140625,
+        "text": " on"
+      },
+      {
+        "id": 573,
+        "logprob": -0.0009536743,
+        "text": " the"
+      },
+      {
+        "id": 26163,
+        "logprob": -0.033203125,
+        "text": " periodic"
+      },
+      {
+        "id": 3037,
+        "logprob": -0.0002670288,
+        "text": " table"
+      },
+      {
+        "id": 235269,
+        "logprob": -4.75,
+        "text": ","
+      },
+      {
+        "id": 7385,
+        "logprob": -11.625,
+        "text": " giving"
+      },
+      {
+        "id": 1853,
+        "logprob": -4.875,
+        "text": " each"
+      },
+      {
+        "id": 5356,
+        "logprob": -0.38867188,
+        "text": " element"
+      },
+      {
+        "id": 1277,
+        "logprob": -3.65625,
+        "text": " its"
+      },
+      {
+        "id": 1997,
+        "logprob": -4.4375,
+        "text": " own"
+      },
+      {
+        "id": 2017,
+        "logprob": -0.29882812,
+        "text": " line"
+      },
+      {
+        "id": 235265,
+        "logprob": -0.16699219,
+        "text": "."
+      },
+      {
+        "id": 107,
+        "logprob": -25.625,
+        "text": "<end_of_turn>"
+      },
+      {
+        "id": 108,
+        "logprob": -6.75,
+        "text": "\n"
+      },
+      {
+        "id": 106,
+        "logprob": -39.5,
+        "text": "<start_of_turn>"
+      },
+      {
+        "id": 2516,
+        "logprob": -32.5,
+        "text": "model"
+      },
+      {
+        "id": 235292,
+        "logprob": -10.125,
+        "text": ":"
+      },
+      {
+        "id": 108,
+        "logprob": -3.421875,
+        "text": "\n"
+      }
+    ],
+    "seed": null,
+    "tokens": [
+      {
+        "id": 688,
+        "logprob": -0.546875,
+        "special": false,
+        "text": "**"
+      },
+      {
+        "id": 103889,
+        "logprob": -0.49023438,
+        "special": false,
+        "text": "Hydrogen"
+      },
+      {
+        "id": 190213,
+        "logprob": -0.48632812,
+        "special": false,
+        "text": "**,"
+      },
+      {
+        "id": 2611,
+        "logprob": -0.58203125,
+        "special": false,
+        "text": " light"
+      },
+      {
+        "id": 578,
+        "logprob": -0.099121094,
+        "special": false,
+        "text": " and"
+      },
+      {
+        "id": 2223,
+        "logprob": -1.078125,
+        "special": false,
+        "text": " free"
+      },
+      {
+        "id": 235269,
+        "logprob": -0.025756836,
+        "special": false,
+        "text": ","
+      },
+      {
+        "id": 108,
+        "logprob": -0.29101562,
+        "special": false,
+        "text": "\n"
+      },
+      {
+        "id": 688,
+        "logprob": -0.0035858154,
+        "special": false,
+        "text": "**"
+      },
+      {
+        "id": 1949,
+        "logprob": -4.1007996e-05,
+        "special": false,
+        "text": "He"
+      }
+    ],
+    "top_tokens": null
+  },
+  "generated_text": "**Hydrogen**, light and free,\n**He"
+}
diff --git a/integration-tests/models/__snapshots__/test_flash_gemma2/test_flash_gemma2_load.json b/integration-tests/models/__snapshots__/test_flash_gemma2/test_flash_gemma2_load.json
new file mode 100644
index 00000000..5c47dd3c
--- /dev/null
+++ b/integration-tests/models/__snapshots__/test_flash_gemma2/test_flash_gemma2_load.json
@@ -0,0 +1,1018 @@
+[
+  {
+    "details": {
+      "best_of_sequences": null,
+      "finish_reason": "length",
+      "generated_tokens": 10,
+      "prefill": [
+        {
+          "id": 2,
+          "logprob": null,
+          "text": "<bos>"
+        },
+        {
+          "id": 106,
+          "logprob": -47.25,
+          "text": "<start_of_turn>"
+        },
+        {
+          "id": 1645,
+          "logprob": -18.875,
+          "text": "user"
+        },
+        {
+          "id": 235292,
+          "logprob": -7.25,
+          "text": ":"
+        },
+        {
+          "id": 108,
+          "logprob": -4.78125,
+          "text": "\n"
+        },
+        {
+          "id": 5559,
+          "logprob": -10.0,
+          "text": "Write"
+        },
+        {
+          "id": 476,
+          "logprob": -0.111816406,
+          "text": " a"
+        },
+        {
+          "id": 19592,
+          "logprob": -2.46875,
+          "text": " poem"
+        },
+        {
+          "id": 577,
+          "logprob": -5.78125,
+          "text": " to"
+        },
+        {
+          "id": 1707,
+          "logprob": -6.375,
+          "text": " help"
+        },
+        {
+          "id": 682,
+          "logprob": -2.125,
+          "text": " me"
+        },
+        {
+          "id": 5434,
+          "logprob": -1.59375,
+          "text": " remember"
+        },
+        {
+          "id": 573,
+          "logprob": -0.62890625,
+          "text": " the"
+        },
+        {
+          "id": 1370,
+          "logprob": -6.625,
+          "text": " first"
+        },
+        {
+          "id": 235248,
+          "logprob": -1.7421875,
+          "text": " "
+        },
+        {
+          "id": 235274,
+          "logprob": -0.44921875,
+          "text": "1"
+        },
+        {
+          "id": 235276,
+          "logprob": -0.07128906,
+          "text": "0"
+        },
+        {
+          "id": 6635,
+          "logprob": -2.109375,
+          "text": " elements"
+        },
+        {
+          "id": 611,
+          "logprob": -0.40429688,
+          "text": " on"
+        },
+        {
+          "id": 573,
+          "logprob": -0.0009918213,
+          "text": " the"
+        },
+        {
+          "id": 26163,
+          "logprob": -0.03540039,
+          "text": " periodic"
+        },
+        {
+          "id": 3037,
+          "logprob": -0.00028800964,
+          "text": " table"
+        },
+        {
+          "id": 235269,
+          "logprob": -4.71875,
+          "text": ","
+        },
+        {
+          "id": 7385,
+          "logprob": -11.875,
+          "text": " giving"
+        },
+        {
+          "id": 1853,
+          "logprob": -4.875,
+          "text": " each"
+        },
+        {
+          "id": 5356,
+          "logprob": -0.38867188,
+          "text": " element"
+        },
+        {
+          "id": 1277,
+          "logprob": -3.65625,
+          "text": " its"
+        },
+        {
+          "id": 1997,
+          "logprob": -4.4375,
+          "text": " own"
+        },
+        {
+          "id": 2017,
+          "logprob": -0.3046875,
+          "text": " line"
+        },
+        {
+          "id": 235265,
+          "logprob": -0.16113281,
+          "text": "."
+        },
+        {
+          "id": 107,
+          "logprob": -25.625,
+          "text": "<end_of_turn>"
+        },
+        {
+          "id": 108,
+          "logprob": -6.75,
+          "text": "\n"
+        },
+        {
+          "id": 106,
+          "logprob": -39.25,
+          "text": "<start_of_turn>"
+        },
+        {
+          "id": 2516,
+          "logprob": -32.5,
+          "text": "model"
+        },
+        {
+          "id": 235292,
+          "logprob": -10.1875,
+          "text": ":"
+        },
+        {
+          "id": 108,
+          "logprob": -3.296875,
+          "text": "\n"
+        }
+      ],
+      "seed": null,
+      "tokens": [
+        {
+          "id": 688,
+          "logprob": -0.546875,
+          "special": false,
+          "text": "**"
+        },
+        {
+          "id": 103889,
+          "logprob": -0.49023438,
+          "special": false,
+          "text": "Hydrogen"
+        },
+        {
+          "id": 190213,
+          "logprob": -0.48632812,
+          "special": false,
+          "text": "**,"
+        },
+        {
+          "id": 2611,
+          "logprob": -0.58203125,
+          "special": false,
+          "text": " light"
+        },
+        {
+          "id": 578,
+          "logprob": -0.08886719,
+          "special": false,
+          "text": " and"
+        },
+        {
+          "id": 2223,
+          "logprob": -1.09375,
+          "special": false,
+          "text": " free"
+        },
+        {
+          "id": 235269,
+          "logprob": -0.024291992,
+          "special": false,
+          "text": ","
+        },
+        {
+          "id": 108,
+          "logprob": -0.30664062,
+          "special": false,
+          "text": "\n"
+        },
+        {
+          "id": 688,
+          "logprob": -0.0035552979,
+          "special": false,
+          "text": "**"
+        },
+        {
+          "id": 1949,
+          "logprob": -4.220009e-05,
+          "special": false,
+          "text": "He"
+        }
+      ],
+      "top_tokens": null
+    },
+    "generated_text": "**Hydrogen**, light and free,\n**He"
+  },
+  {
+    "details": {
+      "best_of_sequences": null,
+      "finish_reason": "length",
+      "generated_tokens": 10,
+      "prefill": [
+        {
+          "id": 2,
+          "logprob": null,
+          "text": "<bos>"
+        },
+        {
+          "id": 106,
+          "logprob": -47.25,
+          "text": "<start_of_turn>"
+        },
+        {
+          "id": 1645,
+          "logprob": -18.875,
+          "text": "user"
+        },
+        {
+          "id": 235292,
+          "logprob": -7.25,
+          "text": ":"
+        },
+        {
+          "id": 108,
+          "logprob": -4.78125,
+          "text": "\n"
+        },
+        {
+          "id": 5559,
+          "logprob": -10.0,
+          "text": "Write"
+        },
+        {
+          "id": 476,
+          "logprob": -0.111816406,
+          "text": " a"
+        },
+        {
+          "id": 19592,
+          "logprob": -2.46875,
+          "text": " poem"
+        },
+        {
+          "id": 577,
+          "logprob": -5.78125,
+          "text": " to"
+        },
+        {
+          "id": 1707,
+          "logprob": -6.375,
+          "text": " help"
+        },
+        {
+          "id": 682,
+          "logprob": -2.125,
+          "text": " me"
+        },
+        {
+          "id": 5434,
+          "logprob": -1.59375,
+          "text": " remember"
+        },
+        {
+          "id": 573,
+          "logprob": -0.62890625,
+          "text": " the"
+        },
+        {
+          "id": 1370,
+          "logprob": -6.625,
+          "text": " first"
+        },
+        {
+          "id": 235248,
+          "logprob": -1.7421875,
+          "text": " "
+        },
+        {
+          "id": 235274,
+          "logprob": -0.44921875,
+          "text": "1"
+        },
+        {
+          "id": 235276,
+          "logprob": -0.07128906,
+          "text": "0"
+        },
+        {
+          "id": 6635,
+          "logprob": -2.109375,
+          "text": " elements"
+        },
+        {
+          "id": 611,
+          "logprob": -0.40429688,
+          "text": " on"
+        },
+        {
+          "id": 573,
+          "logprob": -0.0009918213,
+          "text": " the"
+        },
+        {
+          "id": 26163,
+          "logprob": -0.03540039,
+          "text": " periodic"
+        },
+        {
+          "id": 3037,
+          "logprob": -0.00028800964,
+          "text": " table"
+        },
+        {
+          "id": 235269,
+          "logprob": -4.71875,
+          "text": ","
+        },
+        {
+          "id": 7385,
+          "logprob": -11.875,
+          "text": " giving"
+        },
+        {
+          "id": 1853,
+          "logprob": -4.875,
+          "text": " each"
+        },
+        {
+          "id": 5356,
+          "logprob": -0.38867188,
+          "text": " element"
+        },
+        {
+          "id": 1277,
+          "logprob": -3.65625,
+          "text": " its"
+        },
+        {
+          "id": 1997,
+          "logprob": -4.4375,
+          "text": " own"
+        },
+        {
+          "id": 2017,
+          "logprob": -0.3046875,
+          "text": " line"
+        },
+        {
+          "id": 235265,
+          "logprob": -0.16113281,
+          "text": "."
+        },
+        {
+          "id": 107,
+          "logprob": -25.625,
+          "text": "<end_of_turn>"
+        },
+        {
+          "id": 108,
+          "logprob": -6.75,
+          "text": "\n"
+        },
+        {
+          "id": 106,
+          "logprob": -39.25,
+          "text": "<start_of_turn>"
+        },
+        {
+          "id": 2516,
+          "logprob": -32.5,
+          "text": "model"
+        },
+        {
+          "id": 235292,
+          "logprob": -10.1875,
+          "text": ":"
+        },
+        {
+          "id": 108,
+          "logprob": -3.296875,
+          "text": "\n"
+        }
+      ],
+      "seed": null,
+      "tokens": [
+        {
+          "id": 688,
+          "logprob": -0.546875,
+          "special": false,
+          "text": "**"
+        },
+        {
+          "id": 103889,
+          "logprob": -0.49023438,
+          "special": false,
+          "text": "Hydrogen"
+        },
+        {
+          "id": 190213,
+          "logprob": -0.48632812,
+          "special": false,
+          "text": "**,"
+        },
+        {
+          "id": 2611,
+          "logprob": -0.58203125,
+          "special": false,
+          "text": " light"
+        },
+        {
+          "id": 578,
+          "logprob": -0.08886719,
+          "special": false,
+          "text": " and"
+        },
+        {
+          "id": 2223,
+          "logprob": -1.09375,
+          "special": false,
+          "text": " free"
+        },
+        {
+          "id": 235269,
+          "logprob": -0.024291992,
+          "special": false,
+          "text": ","
+        },
+        {
+          "id": 108,
+          "logprob": -0.30664062,
+          "special": false,
+          "text": "\n"
+        },
+        {
+          "id": 688,
+          "logprob": -0.0035552979,
+          "special": false,
+          "text": "**"
+        },
+        {
+          "id": 1949,
+          "logprob": -4.220009e-05,
+          "special": false,
+          "text": "He"
+        }
+      ],
+      "top_tokens": null
+    },
+    "generated_text": "**Hydrogen**, light and free,\n**He"
+  },
+  {
+    "details": {
+      "best_of_sequences": null,
+      "finish_reason": "length",
+      "generated_tokens": 10,
+      "prefill": [
+        {
+          "id": 2,
+          "logprob": null,
+          "text": "<bos>"
+        },
+        {
+          "id": 106,
+          "logprob": -47.25,
+          "text": "<start_of_turn>"
+        },
+        {
+          "id": 1645,
+          "logprob": -18.875,
+          "text": "user"
+        },
+        {
+          "id": 235292,
+          "logprob": -7.15625,
+          "text": ":"
+        },
+        {
+          "id": 108,
+          "logprob": -4.78125,
+          "text": "\n"
+        },
+        {
+          "id": 5559,
+          "logprob": -10.0,
+          "text": "Write"
+        },
+        {
+          "id": 476,
+          "logprob": -0.1171875,
+          "text": " a"
+        },
+        {
+          "id": 19592,
+          "logprob": -2.46875,
+          "text": " poem"
+        },
+        {
+          "id": 577,
+          "logprob": -5.84375,
+          "text": " to"
+        },
+        {
+          "id": 1707,
+          "logprob": -6.375,
+          "text": " help"
+        },
+        {
+          "id": 682,
+          "logprob": -2.125,
+          "text": " me"
+        },
+        {
+          "id": 5434,
+          "logprob": -1.546875,
+          "text": " remember"
+        },
+        {
+          "id": 573,
+          "logprob": -0.62890625,
+          "text": " the"
+        },
+        {
+          "id": 1370,
+          "logprob": -6.65625,
+          "text": " first"
+        },
+        {
+          "id": 235248,
+          "logprob": -1.84375,
+          "text": " "
+        },
+        {
+          "id": 235274,
+          "logprob": -0.45117188,
+          "text": "1"
+        },
+        {
+          "id": 235276,
+          "logprob": -0.07421875,
+          "text": "0"
+        },
+        {
+          "id": 6635,
+          "logprob": -2.109375,
+          "text": " elements"
+        },
+        {
+          "id": 611,
+          "logprob": -0.4140625,
+          "text": " on"
+        },
+        {
+          "id": 573,
+          "logprob": -0.0009536743,
+          "text": " the"
+        },
+        {
+          "id": 26163,
+          "logprob": -0.033203125,
+          "text": " periodic"
+        },
+        {
+          "id": 3037,
+          "logprob": -0.0002670288,
+          "text": " table"
+        },
+        {
+          "id": 235269,
+          "logprob": -4.75,
+          "text": ","
+        },
+        {
+          "id": 7385,
+          "logprob": -11.625,
+          "text": " giving"
+        },
+        {
+          "id": 1853,
+          "logprob": -4.875,
+          "text": " each"
+        },
+        {
+          "id": 5356,
+          "logprob": -0.38867188,
+          "text": " element"
+        },
+        {
+          "id": 1277,
+          "logprob": -3.65625,
+          "text": " its"
+        },
+        {
+          "id": 1997,
+          "logprob": -4.4375,
+          "text": " own"
+        },
+        {
+          "id": 2017,
+          "logprob": -0.29882812,
+          "text": " line"
+        },
+        {
+          "id": 235265,
+          "logprob": -0.16699219,
+          "text": "."
+        },
+        {
+          "id": 107,
+          "logprob": -25.625,
+          "text": "<end_of_turn>"
+        },
+        {
+          "id": 108,
+          "logprob": -6.75,
+          "text": "\n"
+        },
+        {
+          "id": 106,
+          "logprob": -39.5,
+          "text": "<start_of_turn>"
+        },
+        {
+          "id": 2516,
+          "logprob": -32.5,
+          "text": "model"
+        },
+        {
+          "id": 235292,
+          "logprob": -10.125,
+          "text": ":"
+        },
+        {
+          "id": 108,
+          "logprob": -3.421875,
+          "text": "\n"
+        }
+      ],
+      "seed": null,
+      "tokens": [
+        {
+          "id": 688,
+          "logprob": -0.546875,
+          "special": false,
+          "text": "**"
+        },
+        {
+          "id": 103889,
+          "logprob": -0.49023438,
+          "special": false,
+          "text": "Hydrogen"
+        },
+        {
+          "id": 190213,
+          "logprob": -0.48632812,
+          "special": false,
+          "text": "**,"
+        },
+        {
+          "id": 2611,
+          "logprob": -0.58203125,
+          "special": false,
+          "text": " light"
+        },
+        {
+          "id": 578,
+          "logprob": -0.08984375,
+          "special": false,
+          "text": " and"
+        },
+        {
+          "id": 2223,
+          "logprob": -1.1015625,
+          "special": false,
+          "text": " free"
+        },
+        {
+          "id": 235269,
+          "logprob": -0.024291992,
+          "special": false,
+          "text": ","
+        },
+        {
+          "id": 108,
+          "logprob": -0.30664062,
+          "special": false,
+          "text": "\n"
+        },
+        {
+          "id": 688,
+          "logprob": -0.0038452148,
+          "special": false,
+          "text": "**"
+        },
+        {
+          "id": 1949,
+          "logprob": -4.1484833e-05,
+          "special": false,
+          "text": "He"
+        }
+      ],
+      "top_tokens": null
+    },
+    "generated_text": "**Hydrogen**, light and free,\n**He"
+  },
+  {
+    "details": {
+      "best_of_sequences": null,
+      "finish_reason": "length",
+      "generated_tokens": 10,
+      "prefill": [
+        {
+          "id": 2,
+          "logprob": null,
+          "text": "<bos>"
+        },
+        {
+          "id": 106,
+          "logprob": -47.25,
+          "text": "<start_of_turn>"
+        },
+        {
+          "id": 1645,
+          "logprob": -18.875,
+          "text": "user"
+        },
+        {
+          "id": 235292,
+          "logprob": -7.25,
+          "text": ":"
+        },
+        {
+          "id": 108,
+          "logprob": -4.78125,
+          "text": "\n"
+        },
+        {
+          "id": 5559,
+          "logprob": -10.0,
+          "text": "Write"
+        },
+        {
+          "id": 476,
+          "logprob": -0.111816406,
+          "text": " a"
+        },
+        {
+          "id": 19592,
+          "logprob": -2.46875,
+          "text": " poem"
+        },
+        {
+          "id": 577,
+          "logprob": -5.78125,
+          "text": " to"
+        },
+        {
+          "id": 1707,
+          "logprob": -6.375,
+          "text": " help"
+        },
+        {
+          "id": 682,
+          "logprob": -2.125,
+          "text": " me"
+        },
+        {
+          "id": 5434,
+          "logprob": -1.59375,
+          "text": " remember"
+        },
+        {
+          "id": 573,
+          "logprob": -0.62890625,
+          "text": " the"
+        },
+        {
+          "id": 1370,
+          "logprob": -6.625,
+          "text": " first"
+        },
+        {
+          "id": 235248,
+          "logprob": -1.7421875,
+          "text": " "
+        },
+        {
+          "id": 235274,
+          "logprob": -0.44921875,
+          "text": "1"
+        },
+        {
+          "id": 235276,
+          "logprob": -0.07128906,
+          "text": "0"
+        },
+        {
+          "id": 6635,
+          "logprob": -2.109375,
+          "text": " elements"
+        },
+        {
+          "id": 611,
+          "logprob": -0.40429688,
+          "text": " on"
+        },
+        {
+          "id": 573,
+          "logprob": -0.0009918213,
+          "text": " the"
+        },
+        {
+          "id": 26163,
+          "logprob": -0.03540039,
+          "text": " periodic"
+        },
+        {
+          "id": 3037,
+          "logprob": -0.00028800964,
+          "text": " table"
+        },
+        {
+          "id": 235269,
+          "logprob": -4.71875,
+          "text": ","
+        },
+        {
+          "id": 7385,
+          "logprob": -11.875,
+          "text": " giving"
+        },
+        {
+          "id": 1853,
+          "logprob": -4.875,
+          "text": " each"
+        },
+        {
+          "id": 5356,
+          "logprob": -0.38867188,
+          "text": " element"
+        },
+        {
+          "id": 1277,
+          "logprob": -3.65625,
+          "text": " its"
+        },
+        {
+          "id": 1997,
+          "logprob": -4.4375,
+          "text": " own"
+        },
+        {
+          "id": 2017,
+          "logprob": -0.3046875,
+          "text": " line"
+        },
+        {
+          "id": 235265,
+          "logprob": -0.16113281,
+          "text": "."
+        },
+        {
+          "id": 107,
+          "logprob": -25.625,
+          "text": "<end_of_turn>"
+        },
+        {
+          "id": 108,
+          "logprob": -6.75,
+          "text": "\n"
+        },
+        {
+          "id": 106,
+          "logprob": -39.25,
+          "text": "<start_of_turn>"
+        },
+        {
+          "id": 2516,
+          "logprob": -32.5,
+          "text": "model"
+        },
+        {
+          "id": 235292,
+          "logprob": -10.1875,
+          "text": ":"
+        },
+        {
+          "id": 108,
+          "logprob": -3.296875,
+          "text": "\n"
+        }
+      ],
+      "seed": null,
+      "tokens": [
+        {
+          "id": 688,
+          "logprob": -0.546875,
+          "special": false,
+          "text": "**"
+        },
+        {
+          "id": 103889,
+          "logprob": -0.49023438,
+          "special": false,
+          "text": "Hydrogen"
+        },
+        {
+          "id": 190213,
+          "logprob": -0.48632812,
+          "special": false,
+          "text": "**,"
+        },
+        {
+          "id": 2611,
+          "logprob": -0.58203125,
+          "special": false,
+          "text": " light"
+        },
+        {
+          "id": 578,
+          "logprob": -0.08886719,
+          "special": false,
+          "text": " and"
+        },
+        {
+          "id": 2223,
+          "logprob": -1.09375,
+          "special": false,
+          "text": " free"
+        },
+        {
+          "id": 235269,
+          "logprob": -0.024291992,
+          "special": false,
+          "text": ","
+        },
+        {
+          "id": 108,
+          "logprob": -0.30664062,
+          "special": false,
+          "text": "\n"
+        },
+        {
+          "id": 688,
+          "logprob": -0.0035552979,
+          "special": false,
+          "text": "**"
+        },
+        {
+          "id": 1949,
+          "logprob": -4.220009e-05,
+          "special": false,
+          "text": "He"
+        }
+      ],
+      "top_tokens": null
+    },
+    "generated_text": "**Hydrogen**, light and free,\n**He"
+  }
+]
diff --git a/integration-tests/models/__snapshots__/test_flash_gemma_gptq/test_flash_gemma_gptq.json b/integration-tests/models/__snapshots__/test_flash_gemma_gptq/test_flash_gemma_gptq.json
new file mode 100644
index 00000000..760ebf94
--- /dev/null
+++ b/integration-tests/models/__snapshots__/test_flash_gemma_gptq/test_flash_gemma_gptq.json
@@ -0,0 +1,89 @@
+{
+  "details": {
+    "best_of_sequences": null,
+    "finish_reason": "length",
+    "generated_tokens": 10,
+    "prefill": [
+      {
+        "id": 2,
+        "logprob": null,
+        "text": "<bos>"
+      },
+      {
+        "id": 2015,
+        "logprob": -9.640625,
+        "text": "Test"
+      },
+      {
+        "id": 3853,
+        "logprob": -10.34375,
+        "text": " request"
+      }
+    ],
+    "seed": null,
+    "tokens": [
+      {
+        "id": 604,
+        "logprob": -2.4296875,
+        "special": false,
+        "text": " for"
+      },
+      {
+        "id": 573,
+        "logprob": -2.4453125,
+        "special": false,
+        "text": " the"
+      },
+      {
+        "id": 2412,
+        "logprob": -2.8632812,
+        "special": false,
+        "text": " following"
+      },
+      {
+        "id": 235292,
+        "logprob": -2.1328125,
+        "special": false,
+        "text": ":"
+      },
+      {
+        "id": 109,
+        "logprob": -0.76660156,
+        "special": false,
+        "text": "\n\n"
+      },
+      {
+        "id": 235287,
+        "logprob": -1.3837891,
+        "special": false,
+        "text": "*"
+      },
+      {
+        "id": 235248,
+        "logprob": -1.9746094,
+        "special": false,
+        "text": " "
+      },
+      {
+        "id": 199,
+        "logprob": -1.4189453,
+        "special": false,
+        "text": "<strong>"
+      },
+      {
+        "id": 1232,
+        "logprob": -4.34375,
+        "special": false,
+        "text": "Name"
+      },
+      {
+        "id": 208,
+        "logprob": -0.8852539,
+        "special": false,
+        "text": "</strong>"
+      }
+    ],
+    "top_tokens": null
+  },
+  "generated_text": " for the following:\n\n* <strong>Name</strong>"
+}
diff --git a/integration-tests/models/__snapshots__/test_flash_gemma_gptq/test_flash_gemma_gptq_all_params.json b/integration-tests/models/__snapshots__/test_flash_gemma_gptq/test_flash_gemma_gptq_all_params.json
new file mode 100644
index 00000000..a7019a43
--- /dev/null
+++ b/integration-tests/models/__snapshots__/test_flash_gemma_gptq/test_flash_gemma_gptq_all_params.json
@@ -0,0 +1,89 @@
+{
+  "details": {
+    "best_of_sequences": null,
+    "finish_reason": "length",
+    "generated_tokens": 10,
+    "prefill": [
+      {
+        "id": 2,
+        "logprob": null,
+        "text": "<bos>"
+      },
+      {
+        "id": 2015,
+        "logprob": -9.6484375,
+        "text": "Test"
+      },
+      {
+        "id": 3853,
+        "logprob": -10.3671875,
+        "text": " request"
+      }
+    ],
+    "seed": 0,
+    "tokens": [
+      {
+        "id": 604,
+        "logprob": -0.28271484,
+        "special": false,
+        "text": " for"
+      },
+      {
+        "id": 573,
+        "logprob": -0.18493652,
+        "special": false,
+        "text": " the"
+      },
+      {
+        "id": 16819,
+        "logprob": -1.4804688,
+        "special": false,
+        "text": " detection"
+      },
+      {
+        "id": 576,
+        "logprob": -0.7011719,
+        "special": false,
+        "text": " of"
+      },
+      {
+        "id": 671,
+        "logprob": -2.1738281,
+        "special": false,
+        "text": " an"
+      },
+      {
+        "id": 24646,
+        "logprob": -3.0449219,
+        "special": false,
+        "text": " RNA"
+      },
+      {
+        "id": 12369,
+        "logprob": -0.19299316,
+        "special": false,
+        "text": " virus"
+      },
+      {
+        "id": 575,
+        "logprob": -0.10632324,
+        "special": false,
+        "text": " in"
+      },
+      {
+        "id": 6022,
+        "logprob": -0.98095703,
+        "special": false,
+        "text": " patients"
+      },
+      {
+        "id": 1064,
+        "logprob": -1.3095703,
+        "special": false,
+        "text": " who"
+      }
+    ],
+    "top_tokens": null
+  },
+  "generated_text": "Test request for the detection of an RNA virus in patients who"
+}
diff --git a/integration-tests/models/__snapshots__/test_flash_gemma_gptq/test_flash_gemma_gptq_load.json b/integration-tests/models/__snapshots__/test_flash_gemma_gptq/test_flash_gemma_gptq_load.json
new file mode 100644
index 00000000..bcb9b378
--- /dev/null
+++ b/integration-tests/models/__snapshots__/test_flash_gemma_gptq/test_flash_gemma_gptq_load.json
@@ -0,0 +1,358 @@
+[
+  {
+    "details": {
+      "best_of_sequences": null,
+      "finish_reason": "length",
+      "generated_tokens": 10,
+      "prefill": [
+        {
+          "id": 2,
+          "logprob": null,
+          "text": "<bos>"
+        },
+        {
+          "id": 2015,
+          "logprob": -9.6484375,
+          "text": "Test"
+        },
+        {
+          "id": 3853,
+          "logprob": -10.359375,
+          "text": " request"
+        }
+      ],
+      "seed": null,
+      "tokens": [
+        {
+          "id": 604,
+          "logprob": -2.4277344,
+          "special": false,
+          "text": " for"
+        },
+        {
+          "id": 573,
+          "logprob": -2.4394531,
+          "special": false,
+          "text": " the"
+        },
+        {
+          "id": 2412,
+          "logprob": -2.8613281,
+          "special": false,
+          "text": " following"
+        },
+        {
+          "id": 235292,
+          "logprob": -2.1523438,
+          "special": false,
+          "text": ":"
+        },
+        {
+          "id": 109,
+          "logprob": -0.76220703,
+          "special": false,
+          "text": "\n\n"
+        },
+        {
+          "id": 235287,
+          "logprob": -1.3642578,
+          "special": false,
+          "text": "*"
+        },
+        {
+          "id": 235248,
+          "logprob": -2.0175781,
+          "special": false,
+          "text": " "
+        },
+        {
+          "id": 199,
+          "logprob": -1.4238281,
+          "special": false,
+          "text": "<strong>"
+        },
+        {
+          "id": 1232,
+          "logprob": -4.328125,
+          "special": false,
+          "text": "Name"
+        },
+        {
+          "id": 208,
+          "logprob": -0.8881836,
+          "special": false,
+          "text": "</strong>"
+        }
+      ],
+      "top_tokens": null
+    },
+    "generated_text": " for the following:\n\n* <strong>Name</strong>"
+  },
+  {
+    "details": {
+      "best_of_sequences": null,
+      "finish_reason": "length",
+      "generated_tokens": 10,
+      "prefill": [
+        {
+          "id": 2,
+          "logprob": null,
+          "text": "<bos>"
+        },
+        {
+          "id": 2015,
+          "logprob": -9.6484375,
+          "text": "Test"
+        },
+        {
+          "id": 3853,
+          "logprob": -10.34375,
+          "text": " request"
+        }
+      ],
+      "seed": null,
+      "tokens": [
+        {
+          "id": 604,
+          "logprob": -2.4238281,
+          "special": false,
+          "text": " for"
+        },
+        {
+          "id": 573,
+          "logprob": -2.4453125,
+          "special": false,
+          "text": " the"
+        },
+        {
+          "id": 2412,
+          "logprob": -2.859375,
+          "special": false,
+          "text": " following"
+        },
+        {
+          "id": 235292,
+          "logprob": -2.1445312,
+          "special": false,
+          "text": ":"
+        },
+        {
+          "id": 109,
+          "logprob": -0.7631836,
+          "special": false,
+          "text": "\n\n"
+        },
+        {
+          "id": 235287,
+          "logprob": -1.3642578,
+          "special": false,
+          "text": "*"
+        },
+        {
+          "id": 235248,
+          "logprob": -1.9960938,
+          "special": false,
+          "text": " "
+        },
+        {
+          "id": 199,
+          "logprob": -1.4179688,
+          "special": false,
+          "text": "<strong>"
+        },
+        {
+          "id": 1232,
+          "logprob": -4.3359375,
+          "special": false,
+          "text": "Name"
+        },
+        {
+          "id": 208,
+          "logprob": -0.8847656,
+          "special": false,
+          "text": "</strong>"
+        }
+      ],
+      "top_tokens": null
+    },
+    "generated_text": " for the following:\n\n* <strong>Name</strong>"
+  },
+  {
+    "details": {
+      "best_of_sequences": null,
+      "finish_reason": "length",
+      "generated_tokens": 10,
+      "prefill": [
+        {
+          "id": 2,
+          "logprob": null,
+          "text": "<bos>"
+        },
+        {
+          "id": 2015,
+          "logprob": -9.640625,
+          "text": "Test"
+        },
+        {
+          "id": 3853,
+          "logprob": -10.3671875,
+          "text": " request"
+        }
+      ],
+      "seed": null,
+      "tokens": [
+        {
+          "id": 604,
+          "logprob": -2.4257812,
+          "special": false,
+          "text": " for"
+        },
+        {
+          "id": 573,
+          "logprob": -2.4453125,
+          "special": false,
+          "text": " the"
+        },
+        {
+          "id": 2412,
+          "logprob": -2.8789062,
+          "special": false,
+          "text": " following"
+        },
+        {
+          "id": 235292,
+          "logprob": -2.1367188,
+          "special": false,
+          "text": ":"
+        },
+        {
+          "id": 109,
+          "logprob": -0.76171875,
+          "special": false,
+          "text": "\n\n"
+        },
+        {
+          "id": 235287,
+          "logprob": -1.3515625,
+          "special": false,
+          "text": "*"
+        },
+        {
+          "id": 235248,
+          "logprob": -1.9873047,
+          "special": false,
+          "text": " "
+        },
+        {
+          "id": 199,
+          "logprob": -1.4169922,
+          "special": false,
+          "text": "<strong>"
+        },
+        {
+          "id": 1232,
+          "logprob": -4.3320312,
+          "special": false,
+          "text": "Name"
+        },
+        {
+          "id": 208,
+          "logprob": -0.8930664,
+          "special": false,
+          "text": "</strong>"
+        }
+      ],
+      "top_tokens": null
+    },
+    "generated_text": " for the following:\n\n* <strong>Name</strong>"
+  },
+  {
+    "details": {
+      "best_of_sequences": null,
+      "finish_reason": "length",
+      "generated_tokens": 10,
+      "prefill": [
+        {
+          "id": 2,
+          "logprob": null,
+          "text": "<bos>"
+        },
+        {
+          "id": 2015,
+          "logprob": -9.6484375,
+          "text": "Test"
+        },
+        {
+          "id": 3853,
+          "logprob": -10.359375,
+          "text": " request"
+        }
+      ],
+      "seed": null,
+      "tokens": [
+        {
+          "id": 604,
+          "logprob": -2.4179688,
+          "special": false,
+          "text": " for"
+        },
+        {
+          "id": 573,
+          "logprob": -2.4492188,
+          "special": false,
+          "text": " the"
+        },
+        {
+          "id": 2412,
+          "logprob": -2.8574219,
+          "special": false,
+          "text": " following"
+        },
+        {
+          "id": 235292,
+          "logprob": -2.1445312,
+          "special": false,
+          "text": ":"
+        },
+        {
+          "id": 109,
+          "logprob": -0.7519531,
+          "special": false,
+          "text": "\n\n"
+        },
+        {
+          "id": 235287,
+          "logprob": -1.3623047,
+          "special": false,
+          "text": "*"
+        },
+        {
+          "id": 235248,
+          "logprob": -1.9707031,
+          "special": false,
+          "text": " "
+        },
+        {
+          "id": 199,
+          "logprob": -1.4267578,
+          "special": false,
+          "text": "<strong>"
+        },
+        {
+          "id": 1232,
+          "logprob": -4.3359375,
+          "special": false,
+          "text": "Name"
+        },
+        {
+          "id": 208,
+          "logprob": -0.88427734,
+          "special": false,
+          "text": "</strong>"
+        }
+      ],
+      "top_tokens": null
+    },
+    "generated_text": " for the following:\n\n* <strong>Name</strong>"
+  }
+]
diff --git a/integration-tests/models/__snapshots__/test_flash_llama_exl2/test_flash_llama_exl2.json b/integration-tests/models/__snapshots__/test_flash_llama_exl2/test_flash_llama_exl2.json
new file mode 100644
index 00000000..f6e4bb90
--- /dev/null
+++ b/integration-tests/models/__snapshots__/test_flash_llama_exl2/test_flash_llama_exl2.json
@@ -0,0 +1,84 @@
+{
+  "details": {
+    "best_of_sequences": null,
+    "finish_reason": "length",
+    "generated_tokens": 10,
+    "prefill": [
+      {
+        "id": 2323,
+        "logprob": null,
+        "text": "Test"
+      },
+      {
+        "id": 1715,
+        "logprob": -11.4375,
+        "text": " request"
+      }
+    ],
+    "seed": null,
+    "tokens": [
+      {
+        "id": 25,
+        "logprob": -2.9316406,
+        "special": false,
+        "text": ":"
+      },
+      {
+        "id": 330,
+        "logprob": -3.5136719,
+        "special": false,
+        "text": " \""
+      },
+      {
+        "id": 489,
+        "logprob": -0.7783203,
+        "special": false,
+        "text": " +"
+      },
+      {
+        "id": 1715,
+        "logprob": -1.2314453,
+        "special": false,
+        "text": " request"
+      },
+      {
+        "id": 489,
+        "logprob": -2.0019531,
+        "special": false,
+        "text": " +"
+      },
+      {
+        "id": 2990,
+        "logprob": -1.5009766,
+        "special": false,
+        "text": " \"\\"
+      },
+      {
+        "id": 77,
+        "logprob": -0.057434082,
+        "special": false,
+        "text": "n"
+      },
+      {
+        "id": 702,
+        "logprob": -1.4912109,
+        "special": false,
+        "text": "\"\n"
+      },
+      {
+        "id": 262,
+        "logprob": -1.2636719,
+        "special": false,
+        "text": "   "
+      },
+      {
+        "id": 557,
+        "logprob": -2.4042969,
+        "special": false,
+        "text": " }\n\n"
+      }
+    ],
+    "top_tokens": null
+  },
+  "generated_text": ": \" + request + \"\\n\"\n    }\n\n"
+}
diff --git a/integration-tests/models/__snapshots__/test_flash_llama_exl2/test_flash_llama_exl2_all_params.json b/integration-tests/models/__snapshots__/test_flash_llama_exl2/test_flash_llama_exl2_all_params.json
new file mode 100644
index 00000000..6b38e709
--- /dev/null
+++ b/integration-tests/models/__snapshots__/test_flash_llama_exl2/test_flash_llama_exl2_all_params.json
@@ -0,0 +1,84 @@
+{
+  "details": {
+    "best_of_sequences": null,
+    "finish_reason": "length",
+    "generated_tokens": 10,
+    "prefill": [
+      {
+        "id": 2323,
+        "logprob": null,
+        "text": "Test"
+      },
+      {
+        "id": 1715,
+        "logprob": -11.453125,
+        "text": " request"
+      }
+    ],
+    "seed": 0,
+    "tokens": [
+      {
+        "id": 13,
+        "logprob": -1.9980469,
+        "special": false,
+        "text": "."
+      },
+      {
+        "id": 578,
+        "logprob": -0.15795898,
+        "special": false,
+        "text": " The"
+      },
+      {
+        "id": 3622,
+        "logprob": -1.0458984,
+        "special": false,
+        "text": " server"
+      },
+      {
+        "id": 31680,
+        "logprob": -1.3623047,
+        "special": false,
+        "text": " responds"
+      },
+      {
+        "id": 449,
+        "logprob": 0.0,
+        "special": false,
+        "text": " with"
+      },
+      {
+        "id": 264,
+        "logprob": 0.0,
+        "special": false,
+        "text": " a"
+      },
+      {
+        "id": 330,
+        "logprob": -0.5678711,
+        "special": false,
+        "text": " \""
+      },
+      {
+        "id": 1049,
+        "logprob": -0.12322998,
+        "special": false,
+        "text": "200"
+      },
+      {
+        "id": 10619,
+        "logprob": 0.0,
+        "special": false,
+        "text": " OK"
+      },
+      {
+        "id": 1,
+        "logprob": 0.0,
+        "special": false,
+        "text": "\""
+      }
+    ],
+    "top_tokens": null
+  },
+  "generated_text": "Test request. The server responds with a \"200 OK\""
+}
diff --git a/integration-tests/models/__snapshots__/test_flash_llama_exl2/test_flash_llama_exl2_load.json b/integration-tests/models/__snapshots__/test_flash_llama_exl2/test_flash_llama_exl2_load.json
new file mode 100644
index 00000000..ed369a87
--- /dev/null
+++ b/integration-tests/models/__snapshots__/test_flash_llama_exl2/test_flash_llama_exl2_load.json
@@ -0,0 +1,338 @@
+[
+  {
+    "details": {
+      "best_of_sequences": null,
+      "finish_reason": "length",
+      "generated_tokens": 10,
+      "prefill": [
+        {
+          "id": 2323,
+          "logprob": null,
+          "text": "Test"
+        },
+        {
+          "id": 1715,
+          "logprob": -11.453125,
+          "text": " request"
+        }
+      ],
+      "seed": null,
+      "tokens": [
+        {
+          "id": 25,
+          "logprob": -2.9785156,
+          "special": false,
+          "text": ":"
+        },
+        {
+          "id": 330,
+          "logprob": -3.4941406,
+          "special": false,
+          "text": " \""
+        },
+        {
+          "id": 489,
+          "logprob": -0.79345703,
+          "special": false,
+          "text": " +"
+        },
+        {
+          "id": 1715,
+          "logprob": -1.2324219,
+          "special": false,
+          "text": " request"
+        },
+        {
+          "id": 489,
+          "logprob": -1.9794922,
+          "special": false,
+          "text": " +"
+        },
+        {
+          "id": 2990,
+          "logprob": -1.4892578,
+          "special": false,
+          "text": " \"\\"
+        },
+        {
+          "id": 77,
+          "logprob": -0.058258057,
+          "special": false,
+          "text": "n"
+        },
+        {
+          "id": 702,
+          "logprob": -1.4892578,
+          "special": false,
+          "text": "\"\n"
+        },
+        {
+          "id": 262,
+          "logprob": -1.2783203,
+          "special": false,
+          "text": "   "
+        },
+        {
+          "id": 557,
+          "logprob": -2.3945312,
+          "special": false,
+          "text": " }\n\n"
+        }
+      ],
+      "top_tokens": null
+    },
+    "generated_text": ": \" + request + \"\\n\"\n    }\n\n"
+  },
+  {
+    "details": {
+      "best_of_sequences": null,
+      "finish_reason": "length",
+      "generated_tokens": 10,
+      "prefill": [
+        {
+          "id": 2323,
+          "logprob": null,
+          "text": "Test"
+        },
+        {
+          "id": 1715,
+          "logprob": -11.40625,
+          "text": " request"
+        }
+      ],
+      "seed": null,
+      "tokens": [
+        {
+          "id": 25,
+          "logprob": -2.9433594,
+          "special": false,
+          "text": ":"
+        },
+        {
+          "id": 330,
+          "logprob": -3.4726562,
+          "special": false,
+          "text": " \""
+        },
+        {
+          "id": 489,
+          "logprob": -0.8022461,
+          "special": false,
+          "text": " +"
+        },
+        {
+          "id": 1715,
+          "logprob": -1.2509766,
+          "special": false,
+          "text": " request"
+        },
+        {
+          "id": 489,
+          "logprob": -1.984375,
+          "special": false,
+          "text": " +"
+        },
+        {
+          "id": 2990,
+          "logprob": -1.4677734,
+          "special": false,
+          "text": " \"\\"
+        },
+        {
+          "id": 77,
+          "logprob": -0.059173584,
+          "special": false,
+          "text": "n"
+        },
+        {
+          "id": 702,
+          "logprob": -1.4990234,
+          "special": false,
+          "text": "\"\n"
+        },
+        {
+          "id": 262,
+          "logprob": -1.2822266,
+          "special": false,
+          "text": "   "
+        },
+        {
+          "id": 557,
+          "logprob": -2.3867188,
+          "special": false,
+          "text": " }\n\n"
+        }
+      ],
+      "top_tokens": null
+    },
+    "generated_text": ": \" + request + \"\\n\"\n    }\n\n"
+  },
+  {
+    "details": {
+      "best_of_sequences": null,
+      "finish_reason": "length",
+      "generated_tokens": 10,
+      "prefill": [
+        {
+          "id": 2323,
+          "logprob": null,
+          "text": "Test"
+        },
+        {
+          "id": 1715,
+          "logprob": -11.421875,
+          "text": " request"
+        }
+      ],
+      "seed": null,
+      "tokens": [
+        {
+          "id": 25,
+          "logprob": -2.9511719,
+          "special": false,
+          "text": ":"
+        },
+        {
+          "id": 330,
+          "logprob": -3.46875,
+          "special": false,
+          "text": " \""
+        },
+        {
+          "id": 489,
+          "logprob": -0.77490234,
+          "special": false,
+          "text": " +"
+        },
+        {
+          "id": 1715,
+          "logprob": -1.2558594,
+          "special": false,
+          "text": " request"
+        },
+        {
+          "id": 489,
+          "logprob": -1.984375,
+          "special": false,
+          "text": " +"
+        },
+        {
+          "id": 2990,
+          "logprob": -1.4990234,
+          "special": false,
+          "text": " \"\\"
+        },
+        {
+          "id": 77,
+          "logprob": -0.059143066,
+          "special": false,
+          "text": "n"
+        },
+        {
+          "id": 702,
+          "logprob": -1.4941406,
+          "special": false,
+          "text": "\"\n"
+        },
+        {
+          "id": 262,
+          "logprob": -1.2578125,
+          "special": false,
+          "text": "   "
+        },
+        {
+          "id": 557,
+          "logprob": -2.3964844,
+          "special": false,
+          "text": " }\n\n"
+        }
+      ],
+      "top_tokens": null
+    },
+    "generated_text": ": \" + request + \"\\n\"\n    }\n\n"
+  },
+  {
+    "details": {
+      "best_of_sequences": null,
+      "finish_reason": "length",
+      "generated_tokens": 10,
+      "prefill": [
+        {
+          "id": 2323,
+          "logprob": null,
+          "text": "Test"
+        },
+        {
+          "id": 1715,
+          "logprob": -11.4140625,
+          "text": " request"
+        }
+      ],
+      "seed": null,
+      "tokens": [
+        {
+          "id": 25,
+          "logprob": -2.9101562,
+          "special": false,
+          "text": ":"
+        },
+        {
+          "id": 330,
+          "logprob": -3.5039062,
+          "special": false,
+          "text": " \""
+        },
+        {
+          "id": 489,
+          "logprob": -0.8076172,
+          "special": false,
+          "text": " +"
+        },
+        {
+          "id": 1715,
+          "logprob": -1.2236328,
+          "special": false,
+          "text": " request"
+        },
+        {
+          "id": 489,
+          "logprob": -1.9853516,
+          "special": false,
+          "text": " +"
+        },
+        {
+          "id": 2990,
+          "logprob": -1.4892578,
+          "special": false,
+          "text": " \"\\"
+        },
+        {
+          "id": 77,
+          "logprob": -0.056671143,
+          "special": false,
+          "text": "n"
+        },
+        {
+          "id": 702,
+          "logprob": -1.5107422,
+          "special": false,
+          "text": "\"\n"
+        },
+        {
+          "id": 262,
+          "logprob": -1.2597656,
+          "special": false,
+          "text": "   "
+        },
+        {
+          "id": 557,
+          "logprob": -2.4042969,
+          "special": false,
+          "text": " }\n\n"
+        }
+      ],
+      "top_tokens": null
+    },
+    "generated_text": ": \" + request + \"\\n\"\n    }\n\n"
+  }
+]
diff --git a/integration-tests/models/__snapshots__/test_flash_llama_fp8/test_flash_llama_fp8.json b/integration-tests/models/__snapshots__/test_flash_llama_fp8/test_flash_llama_fp8.json
new file mode 100644
index 00000000..85cfb91f
--- /dev/null
+++ b/integration-tests/models/__snapshots__/test_flash_llama_fp8/test_flash_llama_fp8.json
@@ -0,0 +1,89 @@
+{
+  "details": {
+    "best_of_sequences": null,
+    "finish_reason": "length",
+    "generated_tokens": 10,
+    "prefill": [
+      {
+        "id": 128000,
+        "logprob": null,
+        "text": "<|begin_of_text|>"
+      },
+      {
+        "id": 2323,
+        "logprob": -9.421875,
+        "text": "Test"
+      },
+      {
+        "id": 1715,
+        "logprob": -10.546875,
+        "text": " request"
+      }
+    ],
+    "seed": null,
+    "tokens": [
+      {
+        "id": 369,
+        "logprob": -2.1816406,
+        "special": false,
+        "text": " for"
+      },
+      {
+        "id": 279,
+        "logprob": -2.6992188,
+        "special": false,
+        "text": " the"
+      },
+      {
+        "id": 220,
+        "logprob": -3.6308594,
+        "special": false,
+        "text": " "
+      },
+      {
+        "id": 679,
+        "logprob": -1.7900391,
+        "special": false,
+        "text": "201"
+      },
+      {
+        "id": 24,
+        "logprob": -1.3554688,
+        "special": false,
+        "text": "9"
+      },
+      {
+        "id": 12,
+        "logprob": -2.0039062,
+        "special": false,
+        "text": "-"
+      },
+      {
+        "id": 2366,
+        "logprob": -0.4489746,
+        "special": false,
+        "text": "202"
+      },
+      {
+        "id": 15,
+        "logprob": -0.037109375,
+        "special": false,
+        "text": "0"
+      },
+      {
+        "id": 2978,
+        "logprob": -0.8100586,
+        "special": false,
+        "text": " school"
+      },
+      {
+        "id": 1060,
+        "logprob": -0.013015747,
+        "special": false,
+        "text": " year"
+      }
+    ],
+    "top_tokens": null
+  },
+  "generated_text": " for the 2019-2020 school year"
+}
diff --git a/integration-tests/models/__snapshots__/test_flash_llama_fp8/test_flash_llama_fp8_all_params.json b/integration-tests/models/__snapshots__/test_flash_llama_fp8/test_flash_llama_fp8_all_params.json
new file mode 100644
index 00000000..e39829ec
--- /dev/null
+++ b/integration-tests/models/__snapshots__/test_flash_llama_fp8/test_flash_llama_fp8_all_params.json
@@ -0,0 +1,59 @@
+{
+  "details": {
+    "best_of_sequences": null,
+    "finish_reason": "stop_sequence",
+    "generated_tokens": 5,
+    "prefill": [
+      {
+        "id": 128000,
+        "logprob": null,
+        "text": "<|begin_of_text|>"
+      },
+      {
+        "id": 2323,
+        "logprob": -9.5625,
+        "text": "Test"
+      },
+      {
+        "id": 1715,
+        "logprob": -10.4375,
+        "text": " request"
+      }
+    ],
+    "seed": 0,
+    "tokens": [
+      {
+        "id": 25,
+        "logprob": -0.8984375,
+        "special": false,
+        "text": ":"
+      },
+      {
+        "id": 923,
+        "logprob": -2.84375,
+        "special": false,
+        "text": " add"
+      },
+      {
+        "id": 264,
+        "logprob": 0.0,
+        "special": false,
+        "text": " a"
+      },
+      {
+        "id": 330,
+        "logprob": -0.31640625,
+        "special": false,
+        "text": " \""
+      },
+      {
+        "id": 1985,
+        "logprob": 0.0,
+        "special": false,
+        "text": "test"
+      }
+    ],
+    "top_tokens": null
+  },
+  "generated_text": "Test request: add a \"test"
+}
diff --git a/integration-tests/models/__snapshots__/test_flash_llama_fp8/test_flash_llama_fp8_load.json b/integration-tests/models/__snapshots__/test_flash_llama_fp8/test_flash_llama_fp8_load.json
new file mode 100644
index 00000000..1e9ff6c0
--- /dev/null
+++ b/integration-tests/models/__snapshots__/test_flash_llama_fp8/test_flash_llama_fp8_load.json
@@ -0,0 +1,358 @@
+[
+  {
+    "details": {
+      "best_of_sequences": null,
+      "finish_reason": "length",
+      "generated_tokens": 10,
+      "prefill": [
+        {
+          "id": 128000,
+          "logprob": null,
+          "text": "<|begin_of_text|>"
+        },
+        {
+          "id": 2323,
+          "logprob": -9.5625,
+          "text": "Test"
+        },
+        {
+          "id": 1715,
+          "logprob": -10.375,
+          "text": " request"
+        }
+      ],
+      "seed": null,
+      "tokens": [
+        {
+          "id": 369,
+          "logprob": -2.15625,
+          "special": false,
+          "text": " for"
+        },
+        {
+          "id": 279,
+          "logprob": -2.703125,
+          "special": false,
+          "text": " the"
+        },
+        {
+          "id": 220,
+          "logprob": -3.640625,
+          "special": false,
+          "text": " "
+        },
+        {
+          "id": 679,
+          "logprob": -1.703125,
+          "special": false,
+          "text": "201"
+        },
+        {
+          "id": 24,
+          "logprob": -1.421875,
+          "special": false,
+          "text": "9"
+        },
+        {
+          "id": 12,
+          "logprob": -2.03125,
+          "special": false,
+          "text": "-"
+        },
+        {
+          "id": 2366,
+          "logprob": -0.49023438,
+          "special": false,
+          "text": "202"
+        },
+        {
+          "id": 15,
+          "logprob": -0.041503906,
+          "special": false,
+          "text": "0"
+        },
+        {
+          "id": 2978,
+          "logprob": -0.87109375,
+          "special": false,
+          "text": " school"
+        },
+        {
+          "id": 1060,
+          "logprob": -0.012939453,
+          "special": false,
+          "text": " year"
+        }
+      ],
+      "top_tokens": null
+    },
+    "generated_text": " for the 2019-2020 school year"
+  },
+  {
+    "details": {
+      "best_of_sequences": null,
+      "finish_reason": "length",
+      "generated_tokens": 10,
+      "prefill": [
+        {
+          "id": 128000,
+          "logprob": null,
+          "text": "<|begin_of_text|>"
+        },
+        {
+          "id": 2323,
+          "logprob": -9.5625,
+          "text": "Test"
+        },
+        {
+          "id": 1715,
+          "logprob": -10.375,
+          "text": " request"
+        }
+      ],
+      "seed": null,
+      "tokens": [
+        {
+          "id": 369,
+          "logprob": -2.15625,
+          "special": false,
+          "text": " for"
+        },
+        {
+          "id": 279,
+          "logprob": -2.703125,
+          "special": false,
+          "text": " the"
+        },
+        {
+          "id": 220,
+          "logprob": -3.640625,
+          "special": false,
+          "text": " "
+        },
+        {
+          "id": 679,
+          "logprob": -1.703125,
+          "special": false,
+          "text": "201"
+        },
+        {
+          "id": 24,
+          "logprob": -1.421875,
+          "special": false,
+          "text": "9"
+        },
+        {
+          "id": 12,
+          "logprob": -2.03125,
+          "special": false,
+          "text": "-"
+        },
+        {
+          "id": 2366,
+          "logprob": -0.49023438,
+          "special": false,
+          "text": "202"
+        },
+        {
+          "id": 15,
+          "logprob": -0.041503906,
+          "special": false,
+          "text": "0"
+        },
+        {
+          "id": 2978,
+          "logprob": -0.87109375,
+          "special": false,
+          "text": " school"
+        },
+        {
+          "id": 1060,
+          "logprob": -0.012939453,
+          "special": false,
+          "text": " year"
+        }
+      ],
+      "top_tokens": null
+    },
+    "generated_text": " for the 2019-2020 school year"
+  },
+  {
+    "details": {
+      "best_of_sequences": null,
+      "finish_reason": "length",
+      "generated_tokens": 10,
+      "prefill": [
+        {
+          "id": 128000,
+          "logprob": null,
+          "text": "<|begin_of_text|>"
+        },
+        {
+          "id": 2323,
+          "logprob": -9.5625,
+          "text": "Test"
+        },
+        {
+          "id": 1715,
+          "logprob": -10.375,
+          "text": " request"
+        }
+      ],
+      "seed": null,
+      "tokens": [
+        {
+          "id": 369,
+          "logprob": -2.15625,
+          "special": false,
+          "text": " for"
+        },
+        {
+          "id": 279,
+          "logprob": -2.703125,
+          "special": false,
+          "text": " the"
+        },
+        {
+          "id": 220,
+          "logprob": -3.640625,
+          "special": false,
+          "text": " "
+        },
+        {
+          "id": 679,
+          "logprob": -1.703125,
+          "special": false,
+          "text": "201"
+        },
+        {
+          "id": 24,
+          "logprob": -1.421875,
+          "special": false,
+          "text": "9"
+        },
+        {
+          "id": 12,
+          "logprob": -2.03125,
+          "special": false,
+          "text": "-"
+        },
+        {
+          "id": 2366,
+          "logprob": -0.49023438,
+          "special": false,
+          "text": "202"
+        },
+        {
+          "id": 15,
+          "logprob": -0.041503906,
+          "special": false,
+          "text": "0"
+        },
+        {
+          "id": 2978,
+          "logprob": -0.87109375,
+          "special": false,
+          "text": " school"
+        },
+        {
+          "id": 1060,
+          "logprob": -0.012939453,
+          "special": false,
+          "text": " year"
+        }
+      ],
+      "top_tokens": null
+    },
+    "generated_text": " for the 2019-2020 school year"
+  },
+  {
+    "details": {
+      "best_of_sequences": null,
+      "finish_reason": "length",
+      "generated_tokens": 10,
+      "prefill": [
+        {
+          "id": 128000,
+          "logprob": null,
+          "text": "<|begin_of_text|>"
+        },
+        {
+          "id": 2323,
+          "logprob": -9.5625,
+          "text": "Test"
+        },
+        {
+          "id": 1715,
+          "logprob": -10.375,
+          "text": " request"
+        }
+      ],
+      "seed": null,
+      "tokens": [
+        {
+          "id": 369,
+          "logprob": -2.15625,
+          "special": false,
+          "text": " for"
+        },
+        {
+          "id": 279,
+          "logprob": -2.703125,
+          "special": false,
+          "text": " the"
+        },
+        {
+          "id": 220,
+          "logprob": -3.640625,
+          "special": false,
+          "text": " "
+        },
+        {
+          "id": 679,
+          "logprob": -1.703125,
+          "special": false,
+          "text": "201"
+        },
+        {
+          "id": 24,
+          "logprob": -1.421875,
+          "special": false,
+          "text": "9"
+        },
+        {
+          "id": 12,
+          "logprob": -2.03125,
+          "special": false,
+          "text": "-"
+        },
+        {
+          "id": 2366,
+          "logprob": -0.49023438,
+          "special": false,
+          "text": "202"
+        },
+        {
+          "id": 15,
+          "logprob": -0.041503906,
+          "special": false,
+          "text": "0"
+        },
+        {
+          "id": 2978,
+          "logprob": -0.87109375,
+          "special": false,
+          "text": " school"
+        },
+        {
+          "id": 1060,
+          "logprob": -0.012939453,
+          "special": false,
+          "text": " year"
+        }
+      ],
+      "top_tokens": null
+    },
+    "generated_text": " for the 2019-2020 school year"
+  }
+]
diff --git a/integration-tests/models/__snapshots__/test_flash_llama_gptq/test_flash_llama_gptq.json b/integration-tests/models/__snapshots__/test_flash_llama_gptq/test_flash_llama_gptq.json
index 7797cc6c..0f99d259 100644
--- a/integration-tests/models/__snapshots__/test_flash_llama_gptq/test_flash_llama_gptq.json
+++ b/integration-tests/models/__snapshots__/test_flash_llama_gptq/test_flash_llama_gptq.json
@@ -5,85 +5,80 @@
     "generated_tokens": 10,
     "prefill": [
       {
-        "id": 1,
+        "id": 2323,
         "logprob": null,
-        "text": "<s>"
-      },
-      {
-        "id": 4321,
-        "logprob": -9.7890625,
         "text": "Test"
       },
       {
-        "id": 2009,
-        "logprob": -9.625,
-        "text": "request"
+        "id": 1715,
+        "logprob": -11.34375,
+        "text": " request"
       }
     ],
     "seed": null,
     "tokens": [
       {
-        "id": 13,
-        "logprob": -2.3359375,
+        "id": 198,
+        "logprob": -2.5742188,
         "special": false,
         "text": "\n"
       },
       {
-        "id": 3057,
-        "logprob": -1.8779297,
+        "id": 262,
+        "logprob": -1.6230469,
         "special": false,
-        "text": "Test"
+        "text": "   "
       },
       {
-        "id": 2009,
-        "logprob": -1.2744141,
+        "id": 3270,
+        "logprob": -2.046875,
+        "special": false,
+        "text": " \"\"\"\n"
+      },
+      {
+        "id": 262,
+        "logprob": -0.015281677,
+        "special": false,
+        "text": "   "
+      },
+      {
+        "id": 422,
+        "logprob": -2.1425781,
+        "special": false,
+        "text": " if"
+      },
+      {
+        "id": 1715,
+        "logprob": -0.9238281,
         "special": false,
         "text": " request"
       },
       {
-        "id": 13,
-        "logprob": -1.6933594,
+        "id": 13204,
+        "logprob": -0.076660156,
         "special": false,
-        "text": "\n"
+        "text": ".method"
       },
       {
-        "id": 3057,
-        "logprob": -1.4648438,
+        "id": 624,
+        "logprob": -0.021987915,
         "special": false,
-        "text": "Test"
+        "text": " =="
       },
       {
-        "id": 2009,
-        "logprob": -0.15600586,
+        "id": 364,
+        "logprob": -0.39208984,
         "special": false,
-        "text": " request"
+        "text": " '"
       },
       {
-        "id": 13,
-        "logprob": -0.8027344,
+        "id": 3019,
+        "logprob": -0.10821533,
         "special": false,
-        "text": "\n"
-      },
-      {
-        "id": 3057,
-        "logprob": -0.23022461,
-        "special": false,
-        "text": "Test"
-      },
-      {
-        "id": 2009,
-        "logprob": -0.0069885254,
-        "special": false,
-        "text": " request"
-      },
-      {
-        "id": 13,
-        "logprob": -0.02218628,
-        "special": false,
-        "text": "\n"
+        "text": "POST"
       }
     ],
     "top_tokens": null
   },
-  "generated_text": "\nTest request\nTest request\nTest request\n"
+  "generated_text": "\n    \"\"\"\n    if request.method == 'POST"
 }
diff --git a/integration-tests/models/__snapshots__/test_flash_llama_gptq/test_flash_llama_gptq_all_params.json b/integration-tests/models/__snapshots__/test_flash_llama_gptq/test_flash_llama_gptq_all_params.json
index fa2fd4a2..4152b5b3 100644
--- a/integration-tests/models/__snapshots__/test_flash_llama_gptq/test_flash_llama_gptq_all_params.json
+++ b/integration-tests/models/__snapshots__/test_flash_llama_gptq/test_flash_llama_gptq_all_params.json
@@ -5,85 +5,80 @@
     "generated_tokens": 10,
     "prefill": [
       {
-        "id": 1,
+        "id": 2323,
         "logprob": null,
-        "text": "<s>"
-      },
-      {
-        "id": 4321,
-        "logprob": -9.84375,
         "text": "Test"
       },
       {
-        "id": 2009,
-        "logprob": -9.6015625,
-        "text": "request"
+        "id": 1715,
+        "logprob": -11.34375,
+        "text": " request"
       }
     ],
     "seed": 0,
     "tokens": [
       {
-        "id": 29899,
-        "logprob": -1.5625,
+        "id": 13,
+        "logprob": -2.2539062,
         "special": false,
-        "text": "-"
+        "text": "."
       },
       {
-        "id": 1454,
-        "logprob": -0.20410156,
+        "id": 578,
+        "logprob": -0.15563965,
         "special": false,
-        "text": "for"
+        "text": " The"
       },
       {
-        "id": 29899,
+        "id": 3622,
+        "logprob": -0.8203125,
+        "special": false,
+        "text": " server"
+      },
+      {
+        "id": 706,
         "logprob": 0.0,
         "special": false,
-        "text": "-"
+        "text": " has"
       },
       {
-        "id": 9342,
+        "id": 539,
         "logprob": 0.0,
         "special": false,
-        "text": "comment"
+        "text": " not"
       },
       {
-        "id": 29901,
+        "id": 3686,
         "logprob": 0.0,
         "special": false,
-        "text": ":"
+        "text": " yet"
       },
       {
-        "id": 396,
-        "logprob": -0.27685547,
-        "special": false,
-        "text": " #"
-      },
-      {
-        "id": 29906,
-        "logprob": -0.4970703,
-        "special": false,
-        "text": "2"
-      },
-      {
-        "id": 29900,
-        "logprob": -0.80615234,
-        "special": false,
-        "text": "0"
-      },
-      {
-        "id": 29896,
+        "id": 3288,
         "logprob": 0.0,
         "special": false,
-        "text": "1"
+        "text": " sent"
       },
       {
-        "id": 29955,
-        "logprob": -1.0751953,
+        "id": 904,
+        "logprob": 0.0,
         "special": false,
-        "text": "7"
+        "text": " any"
+      },
+      {
+        "id": 828,
+        "logprob": 0.0,
+        "special": false,
+        "text": " data"
+      },
+      {
+        "id": 382,
+        "logprob": -1.5517578,
+        "special": false,
+        "text": ".\n\n"
       }
     ],
     "top_tokens": null
   },
-  "generated_text": "Test request-for-comment: #2017"
+  "generated_text": "Test request. The server has not yet sent any data.\n\n"
 }
diff --git a/integration-tests/models/__snapshots__/test_flash_llama_gptq/test_flash_llama_gptq_load.json b/integration-tests/models/__snapshots__/test_flash_llama_gptq/test_flash_llama_gptq_load.json
index 594b7351..75e90303 100644
--- a/integration-tests/models/__snapshots__/test_flash_llama_gptq/test_flash_llama_gptq_load.json
+++ b/integration-tests/models/__snapshots__/test_flash_llama_gptq/test_flash_llama_gptq_load.json
@@ -6,87 +6,82 @@
       "generated_tokens": 10,
       "prefill": [
         {
-          "id": 1,
+          "id": 2323,
           "logprob": null,
-          "text": "<s>"
-        },
-        {
-          "id": 4321,
-          "logprob": -9.828125,
           "text": "Test"
         },
         {
-          "id": 2009,
-          "logprob": -9.609375,
-          "text": "request"
+          "id": 1715,
+          "logprob": -11.34375,
+          "text": " request"
         }
       ],
       "seed": null,
       "tokens": [
         {
-          "id": 13,
-          "logprob": -2.3300781,
+          "id": 198,
+          "logprob": -2.5742188,
           "special": false,
           "text": "\n"
         },
         {
-          "id": 3057,
-          "logprob": -1.8740234,
+          "id": 262,
+          "logprob": -1.6220703,
           "special": false,
-          "text": "Test"
+          "text": "   "
         },
         {
-          "id": 2009,
-          "logprob": -1.2646484,
+          "id": 3270,
+          "logprob": -2.0410156,
+          "special": false,
+          "text": " \"\"\"\n"
+        },
+        {
+          "id": 262,
+          "logprob": -0.015281677,
+          "special": false,
+          "text": "   "
+        },
+        {
+          "id": 422,
+          "logprob": -2.1445312,
+          "special": false,
+          "text": " if"
+        },
+        {
+          "id": 1715,
+          "logprob": -0.92333984,
           "special": false,
           "text": " request"
         },
         {
-          "id": 13,
-          "logprob": -1.7158203,
+          "id": 13204,
+          "logprob": -0.07672119,
           "special": false,
-          "text": "\n"
+          "text": ".method"
         },
         {
-          "id": 3057,
-          "logprob": -1.4667969,
+          "id": 624,
+          "logprob": -0.021987915,
           "special": false,
-          "text": "Test"
+          "text": " =="
         },
         {
-          "id": 2009,
-          "logprob": -0.15344238,
+          "id": 364,
+          "logprob": -0.39208984,
           "special": false,
-          "text": " request"
+          "text": " '"
         },
         {
-          "id": 13,
-          "logprob": -0.81591797,
+          "id": 3019,
+          "logprob": -0.10638428,
           "special": false,
-          "text": "\n"
-        },
-        {
-          "id": 3057,
-          "logprob": -0.22973633,
-          "special": false,
-          "text": "Test"
-        },
-        {
-          "id": 2009,
-          "logprob": -0.007045746,
-          "special": false,
-          "text": " request"
-        },
-        {
-          "id": 13,
-          "logprob": -0.021957397,
-          "special": false,
-          "text": "\n"
+          "text": "POST"
         }
       ],
       "top_tokens": null
     },
-    "generated_text": "\nTest request\nTest request\nTest request\n"
+    "generated_text": "\n    \"\"\"\n    if request.method == 'POST"
   },
   {
     "details": {
@@ -95,87 +90,82 @@
       "generated_tokens": 10,
       "prefill": [
         {
-          "id": 1,
+          "id": 2323,
           "logprob": null,
-          "text": "<s>"
-        },
-        {
-          "id": 4321,
-          "logprob": -9.84375,
           "text": "Test"
         },
         {
-          "id": 2009,
-          "logprob": -9.59375,
-          "text": "request"
+          "id": 1715,
+          "logprob": -11.34375,
+          "text": " request"
         }
       ],
       "seed": null,
       "tokens": [
         {
-          "id": 13,
-          "logprob": -2.3378906,
+          "id": 198,
+          "logprob": -2.5742188,
           "special": false,
           "text": "\n"
         },
         {
-          "id": 3057,
-          "logprob": -1.8779297,
+          "id": 262,
+          "logprob": -1.6220703,
           "special": false,
-          "text": "Test"
+          "text": "   "
         },
         {
-          "id": 2009,
-          "logprob": -1.2636719,
+          "id": 3270,
+          "logprob": -2.0410156,
+          "special": false,
+          "text": " \"\"\"\n"
+        },
+        {
+          "id": 262,
+          "logprob": -0.015281677,
+          "special": false,
+          "text": "   "
+        },
+        {
+          "id": 422,
+          "logprob": -2.1445312,
+          "special": false,
+          "text": " if"
+        },
+        {
+          "id": 1715,
+          "logprob": -0.92333984,
           "special": false,
           "text": " request"
         },
         {
-          "id": 13,
-          "logprob": -1.6992188,
+          "id": 13204,
+          "logprob": -0.07672119,
           "special": false,
-          "text": "\n"
+          "text": ".method"
         },
         {
-          "id": 3057,
-          "logprob": -1.4589844,
+          "id": 624,
+          "logprob": -0.021987915,
           "special": false,
-          "text": "Test"
+          "text": " =="
         },
         {
-          "id": 2009,
-          "logprob": -0.15344238,
+          "id": 364,
+          "logprob": -0.39208984,
           "special": false,
-          "text": " request"
+          "text": " '"
         },
         {
-          "id": 13,
-          "logprob": -0.79052734,
+          "id": 3019,
+          "logprob": -0.10638428,
           "special": false,
-          "text": "\n"
-        },
-        {
-          "id": 3057,
-          "logprob": -0.22937012,
-          "special": false,
-          "text": "Test"
-        },
-        {
-          "id": 2009,
-          "logprob": -0.007041931,
-          "special": false,
-          "text": " request"
-        },
-        {
-          "id": 13,
-          "logprob": -0.022140503,
-          "special": false,
-          "text": "\n"
+          "text": "POST"
         }
       ],
       "top_tokens": null
     },
-    "generated_text": "\nTest request\nTest request\nTest request\n"
+    "generated_text": "\n    \"\"\"\n    if request.method == 'POST"
   },
   {
     "details": {
@@ -184,87 +174,82 @@
       "generated_tokens": 10,
       "prefill": [
         {
-          "id": 1,
+          "id": 2323,
           "logprob": null,
-          "text": "<s>"
-        },
-        {
-          "id": 4321,
-          "logprob": -9.84375,
           "text": "Test"
         },
         {
-          "id": 2009,
-          "logprob": -9.609375,
-          "text": "request"
+          "id": 1715,
+          "logprob": -11.34375,
+          "text": " request"
         }
       ],
       "seed": null,
       "tokens": [
         {
-          "id": 13,
-          "logprob": -2.3261719,
+          "id": 198,
+          "logprob": -2.5742188,
           "special": false,
           "text": "\n"
         },
         {
-          "id": 3057,
-          "logprob": -1.8730469,
+          "id": 262,
+          "logprob": -1.6220703,
           "special": false,
-          "text": "Test"
+          "text": "   "
         },
         {
-          "id": 2009,
-          "logprob": -1.2587891,
+          "id": 3270,
+          "logprob": -2.0410156,
+          "special": false,
+          "text": " \"\"\"\n"
+        },
+        {
+          "id": 262,
+          "logprob": -0.015281677,
+          "special": false,
+          "text": "   "
+        },
+        {
+          "id": 422,
+          "logprob": -2.1445312,
+          "special": false,
+          "text": " if"
+        },
+        {
+          "id": 1715,
+          "logprob": -0.92333984,
           "special": false,
           "text": " request"
         },
         {
-          "id": 13,
-          "logprob": -1.6894531,
+          "id": 13204,
+          "logprob": -0.07672119,
           "special": false,
-          "text": "\n"
+          "text": ".method"
         },
         {
-          "id": 3057,
-          "logprob": -1.46875,
+          "id": 624,
+          "logprob": -0.021987915,
           "special": false,
-          "text": "Test"
+          "text": " =="
         },
         {
-          "id": 2009,
-          "logprob": -0.1541748,
+          "id": 364,
+          "logprob": -0.39208984,
           "special": false,
-          "text": " request"
+          "text": " '"
         },
         {
-          "id": 13,
-          "logprob": -0.80322266,
+          "id": 3019,
+          "logprob": -0.10638428,
           "special": false,
-          "text": "\n"
-        },
-        {
-          "id": 3057,
-          "logprob": -0.22912598,
-          "special": false,
-          "text": "Test"
-        },
-        {
-          "id": 2009,
-          "logprob": -0.0070495605,
-          "special": false,
-          "text": " request"
-        },
-        {
-          "id": 13,
-          "logprob": -0.021606445,
-          "special": false,
-          "text": "\n"
+          "text": "POST"
         }
       ],
       "top_tokens": null
     },
-    "generated_text": "\nTest request\nTest request\nTest request\n"
+    "generated_text": "\n    \"\"\"\n    if request.method == 'POST"
   },
   {
     "details": {
@@ -273,86 +258,81 @@
       "generated_tokens": 10,
       "prefill": [
         {
-          "id": 1,
+          "id": 2323,
           "logprob": null,
-          "text": "<s>"
-        },
-        {
-          "id": 4321,
-          "logprob": -9.84375,
           "text": "Test"
         },
         {
-          "id": 2009,
-          "logprob": -9.6015625,
-          "text": "request"
+          "id": 1715,
+          "logprob": -11.34375,
+          "text": " request"
         }
       ],
       "seed": null,
       "tokens": [
         {
-          "id": 13,
-          "logprob": -2.3320312,
+          "id": 198,
+          "logprob": -2.5742188,
           "special": false,
           "text": "\n"
         },
         {
-          "id": 3057,
-          "logprob": -1.875,
+          "id": 262,
+          "logprob": -1.6220703,
           "special": false,
-          "text": "Test"
+          "text": "   "
         },
         {
-          "id": 2009,
-          "logprob": -1.2646484,
+          "id": 3270,
+          "logprob": -2.0410156,
+          "special": false,
+          "text": " \"\"\"\n"
+        },
+        {
+          "id": 262,
+          "logprob": -0.015281677,
+          "special": false,
+          "text": "   "
+        },
+        {
+          "id": 422,
+          "logprob": -2.1445312,
+          "special": false,
+          "text": " if"
+        },
+        {
+          "id": 1715,
+          "logprob": -0.92333984,
           "special": false,
           "text": " request"
         },
         {
-          "id": 13,
-          "logprob": -1.6884766,
+          "id": 13204,
+          "logprob": -0.07672119,
           "special": false,
-          "text": "\n"
+          "text": ".method"
         },
         {
-          "id": 3057,
-          "logprob": -1.4589844,
+          "id": 624,
+          "logprob": -0.021987915,
           "special": false,
-          "text": "Test"
+          "text": " =="
         },
         {
-          "id": 2009,
-          "logprob": -0.15185547,
+          "id": 364,
+          "logprob": -0.39208984,
           "special": false,
-          "text": " request"
+          "text": " '"
         },
         {
-          "id": 13,
-          "logprob": -0.79833984,
+          "id": 3019,
+          "logprob": -0.10638428,
           "special": false,
-          "text": "\n"
-        },
-        {
-          "id": 3057,
-          "logprob": -0.22827148,
-          "special": false,
-          "text": "Test"
-        },
-        {
-          "id": 2009,
-          "logprob": -0.006996155,
-          "special": false,
-          "text": " request"
-        },
-        {
-          "id": 13,
-          "logprob": -0.021560669,
-          "special": false,
-          "text": "\n"
+          "text": "POST"
         }
       ],
       "top_tokens": null
     },
-    "generated_text": "\nTest request\nTest request\nTest request\n"
+    "generated_text": "\n    \"\"\"\n    if request.method == 'POST"
   }
 ]
diff --git a/integration-tests/models/__snapshots__/test_flash_llama_marlin/test_flash_llama_marlin.json b/integration-tests/models/__snapshots__/test_flash_llama_marlin/test_flash_llama_marlin.json
new file mode 100644
index 00000000..47849a3f
--- /dev/null
+++ b/integration-tests/models/__snapshots__/test_flash_llama_marlin/test_flash_llama_marlin.json
@@ -0,0 +1,89 @@
+{
+  "details": {
+    "best_of_sequences": null,
+    "finish_reason": "length",
+    "generated_tokens": 10,
+    "prefill": [
+      {
+        "id": 1,
+        "logprob": null,
+        "text": "<s>"
+      },
+      {
+        "id": 4321,
+        "logprob": -12.390625,
+        "text": "Test"
+      },
+      {
+        "id": 2009,
+        "logprob": -11.0625,
+        "text": "request"
+      }
+    ],
+    "seed": null,
+    "tokens": [
+      {
+        "id": 13,
+        "logprob": -2.0507812,
+        "special": false,
+        "text": "\n"
+      },
+      {
+        "id": 13,
+        "logprob": -2.3007812,
+        "special": false,
+        "text": "\n"
+      },
+      {
+        "id": 29902,
+        "logprob": -2.0449219,
+        "special": false,
+        "text": "I"
+      },
+      {
+        "id": 505,
+        "logprob": -1.3242188,
+        "special": false,
+        "text": " have"
+      },
+      {
+        "id": 263,
+        "logprob": -0.2076416,
+        "special": false,
+        "text": " a"
+      },
+      {
+        "id": 1243,
+        "logprob": -2.0273438,
+        "special": false,
+        "text": " test"
+      },
+      {
+        "id": 2009,
+        "logprob": -0.6845703,
+        "special": false,
+        "text": " request"
+      },
+      {
+        "id": 515,
+        "logprob": -1.1748047,
+        "special": false,
+        "text": " from"
+      },
+      {
+        "id": 263,
+        "logprob": -1.0644531,
+        "special": false,
+        "text": " a"
+      },
+      {
+        "id": 1404,
+        "logprob": -1.5224609,
+        "special": false,
+        "text": " user"
+      }
+    ],
+    "top_tokens": null
+  },
+  "generated_text": "\n\nI have a test request from a user"
+}
diff --git a/integration-tests/models/__snapshots__/test_flash_llama_marlin/test_flash_llama_marlin_all_params.json b/integration-tests/models/__snapshots__/test_flash_llama_marlin/test_flash_llama_marlin_all_params.json
new file mode 100644
index 00000000..bda2393e
--- /dev/null
+++ b/integration-tests/models/__snapshots__/test_flash_llama_marlin/test_flash_llama_marlin_all_params.json
@@ -0,0 +1,89 @@
+{
+  "details": {
+    "best_of_sequences": null,
+    "finish_reason": "length",
+    "generated_tokens": 10,
+    "prefill": [
+      {
+        "id": 1,
+        "logprob": null,
+        "text": "<s>"
+      },
+      {
+        "id": 4321,
+        "logprob": -12.390625,
+        "text": "Test"
+      },
+      {
+        "id": 2009,
+        "logprob": -11.0625,
+        "text": "request"
+      }
+    ],
+    "seed": 0,
+    "tokens": [
+      {
+        "id": 5229,
+        "logprob": -1.2607422,
+        "special": false,
+        "text": " failed"
+      },
+      {
+        "id": 29901,
+        "logprob": 0.0,
+        "special": false,
+        "text": ":"
+      },
+      {
+        "id": 6527,
+        "logprob": -0.11450195,
+        "special": false,
+        "text": " Could"
+      },
+      {
+        "id": 451,
+        "logprob": 0.0,
+        "special": false,
+        "text": " not"
+      },
+      {
+        "id": 4511,
+        "logprob": -0.2286377,
+        "special": false,
+        "text": " connect"
+      },
+      {
+        "id": 304,
+        "logprob": 0.0,
+        "special": false,
+        "text": " to"
+      },
+      {
+        "id": 1923,
+        "logprob": -1.2568359,
+        "special": false,
+        "text": " server"
+      },
+      {
+        "id": 13,
+        "logprob": 0.0,
+        "special": false,
+        "text": "\n"
+      },
+      {
+        "id": 13,
+        "logprob": -0.15905762,
+        "special": false,
+        "text": "\n"
+      },
+      {
+        "id": 29902,
+        "logprob": -0.21618652,
+        "special": false,
+        "text": "I"
+      }
+    ],
+    "top_tokens": null
+  },
+  "generated_text": "Test request failed: Could not connect to server\n\nI"
+}
diff --git a/integration-tests/models/__snapshots__/test_flash_llama_marlin/test_flash_llama_marlin_load.json b/integration-tests/models/__snapshots__/test_flash_llama_marlin/test_flash_llama_marlin_load.json
new file mode 100644
index 00000000..44c26efb
--- /dev/null
+++ b/integration-tests/models/__snapshots__/test_flash_llama_marlin/test_flash_llama_marlin_load.json
@@ -0,0 +1,358 @@
+[
+  {
+    "details": {
+      "best_of_sequences": null,
+      "finish_reason": "length",
+      "generated_tokens": 10,
+      "prefill": [
+        {
+          "id": 1,
+          "logprob": null,
+          "text": "<s>"
+        },
+        {
+          "id": 4321,
+          "logprob": -12.390625,
+          "text": "Test"
+        },
+        {
+          "id": 2009,
+          "logprob": -11.0625,
+          "text": "request"
+        }
+      ],
+      "seed": null,
+      "tokens": [
+        {
+          "id": 13,
+          "logprob": -2.0507812,
+          "special": false,
+          "text": "\n"
+        },
+        {
+          "id": 13,
+          "logprob": -2.3007812,
+          "special": false,
+          "text": "\n"
+        },
+        {
+          "id": 29902,
+          "logprob": -2.0449219,
+          "special": false,
+          "text": "I"
+        },
+        {
+          "id": 505,
+          "logprob": -1.3242188,
+          "special": false,
+          "text": " have"
+        },
+        {
+          "id": 263,
+          "logprob": -0.2076416,
+          "special": false,
+          "text": " a"
+        },
+        {
+          "id": 1243,
+          "logprob": -2.0273438,
+          "special": false,
+          "text": " test"
+        },
+        {
+          "id": 2009,
+          "logprob": -0.6845703,
+          "special": false,
+          "text": " request"
+        },
+        {
+          "id": 515,
+          "logprob": -1.1748047,
+          "special": false,
+          "text": " from"
+        },
+        {
+          "id": 263,
+          "logprob": -1.0595703,
+          "special": false,
+          "text": " a"
+        },
+        {
+          "id": 1404,
+          "logprob": -1.5224609,
+          "special": false,
+          "text": " user"
+        }
+      ],
+      "top_tokens": null
+    },
+    "generated_text": "\n\nI have a test request from a user"
+  },
+  {
+    "details": {
+      "best_of_sequences": null,
+      "finish_reason": "length",
+      "generated_tokens": 10,
+      "prefill": [
+        {
+          "id": 1,
+          "logprob": null,
+          "text": "<s>"
+        },
+        {
+          "id": 4321,
+          "logprob": -12.390625,
+          "text": "Test"
+        },
+        {
+          "id": 2009,
+          "logprob": -11.0625,
+          "text": "request"
+        }
+      ],
+      "seed": null,
+      "tokens": [
+        {
+          "id": 13,
+          "logprob": -2.0507812,
+          "special": false,
+          "text": "\n"
+        },
+        {
+          "id": 13,
+          "logprob": -2.3007812,
+          "special": false,
+          "text": "\n"
+        },
+        {
+          "id": 29902,
+          "logprob": -2.0449219,
+          "special": false,
+          "text": "I"
+        },
+        {
+          "id": 505,
+          "logprob": -1.3242188,
+          "special": false,
+          "text": " have"
+        },
+        {
+          "id": 263,
+          "logprob": -0.2076416,
+          "special": false,
+          "text": " a"
+        },
+        {
+          "id": 1243,
+          "logprob": -2.0273438,
+          "special": false,
+          "text": " test"
+        },
+        {
+          "id": 2009,
+          "logprob": -0.6845703,
+          "special": false,
+          "text": " request"
+        },
+        {
+          "id": 515,
+          "logprob": -1.1748047,
+          "special": false,
+          "text": " from"
+        },
+        {
+          "id": 263,
+          "logprob": -1.0595703,
+          "special": false,
+          "text": " a"
+        },
+        {
+          "id": 1404,
+          "logprob": -1.5224609,
+          "special": false,
+          "text": " user"
+        }
+      ],
+      "top_tokens": null
+    },
+    "generated_text": "\n\nI have a test request from a user"
+  },
+  {
+    "details": {
+      "best_of_sequences": null,
+      "finish_reason": "length",
+      "generated_tokens": 10,
+      "prefill": [
+        {
+          "id": 1,
+          "logprob": null,
+          "text": "<s>"
+        },
+        {
+          "id": 4321,
+          "logprob": -12.390625,
+          "text": "Test"
+        },
+        {
+          "id": 2009,
+          "logprob": -11.0625,
+          "text": "request"
+        }
+      ],
+      "seed": null,
+      "tokens": [
+        {
+          "id": 13,
+          "logprob": -2.0507812,
+          "special": false,
+          "text": "\n"
+        },
+        {
+          "id": 13,
+          "logprob": -2.3007812,
+          "special": false,
+          "text": "\n"
+        },
+        {
+          "id": 29902,
+          "logprob": -2.0449219,
+          "special": false,
+          "text": "I"
+        },
+        {
+          "id": 505,
+          "logprob": -1.3242188,
+          "special": false,
+          "text": " have"
+        },
+        {
+          "id": 263,
+          "logprob": -0.2076416,
+          "special": false,
+          "text": " a"
+        },
+        {
+          "id": 1243,
+          "logprob": -2.0273438,
+          "special": false,
+          "text": " test"
+        },
+        {
+          "id": 2009,
+          "logprob": -0.6845703,
+          "special": false,
+          "text": " request"
+        },
+        {
+          "id": 515,
+          "logprob": -1.1748047,
+          "special": false,
+          "text": " from"
+        },
+        {
+          "id": 263,
+          "logprob": -1.0595703,
+          "special": false,
+          "text": " a"
+        },
+        {
+          "id": 1404,
+          "logprob": -1.5224609,
+          "special": false,
+          "text": " user"
+        }
+      ],
+      "top_tokens": null
+    },
+    "generated_text": "\n\nI have a test request from a user"
+  },
+  {
+    "details": {
+      "best_of_sequences": null,
+      "finish_reason": "length",
+      "generated_tokens": 10,
+      "prefill": [
+        {
+          "id": 1,
+          "logprob": null,
+          "text": "<s>"
+        },
+        {
+          "id": 4321,
+          "logprob": -12.390625,
+          "text": "Test"
+        },
+        {
+          "id": 2009,
+          "logprob": -11.0625,
+          "text": "request"
+        }
+      ],
+      "seed": null,
+      "tokens": [
+        {
+          "id": 13,
+          "logprob": -2.0507812,
+          "special": false,
+          "text": "\n"
+        },
+        {
+          "id": 13,
+          "logprob": -2.3007812,
+          "special": false,
+          "text": "\n"
+        },
+        {
+          "id": 29902,
+          "logprob": -2.0449219,
+          "special": false,
+          "text": "I"
+        },
+        {
+          "id": 505,
+          "logprob": -1.3242188,
+          "special": false,
+          "text": " have"
+        },
+        {
+          "id": 263,
+          "logprob": -0.2076416,
+          "special": false,
+          "text": " a"
+        },
+        {
+          "id": 1243,
+          "logprob": -2.0273438,
+          "special": false,
+          "text": " test"
+        },
+        {
+          "id": 2009,
+          "logprob": -0.6845703,
+          "special": false,
+          "text": " request"
+        },
+        {
+          "id": 515,
+          "logprob": -1.1748047,
+          "special": false,
+          "text": " from"
+        },
+        {
+          "id": 263,
+          "logprob": -1.0595703,
+          "special": false,
+          "text": " a"
+        },
+        {
+          "id": 1404,
+          "logprob": -1.5224609,
+          "special": false,
+          "text": " user"
+        }
+      ],
+      "top_tokens": null
+    },
+    "generated_text": "\n\nI have a test request from a user"
+  }
+]
diff --git a/integration-tests/models/__snapshots__/test_flash_llama_marlin_24/test_flash_llama_marlin.json b/integration-tests/models/__snapshots__/test_flash_llama_marlin_24/test_flash_llama_marlin.json
new file mode 100644
index 00000000..94883de5
--- /dev/null
+++ b/integration-tests/models/__snapshots__/test_flash_llama_marlin_24/test_flash_llama_marlin.json
@@ -0,0 +1,89 @@
+{
+  "details": {
+    "best_of_sequences": null,
+    "finish_reason": "length",
+    "generated_tokens": 10,
+    "prefill": [
+      {
+        "id": 1,
+        "logprob": null,
+        "text": "<s>"
+      },
+      {
+        "id": 4321,
+        "logprob": -9.0859375,
+        "text": "Test"
+      },
+      {
+        "id": 2009,
+        "logprob": -16.359375,
+        "text": "request"
+      }
+    ],
+    "seed": null,
+    "tokens": [
+      {
+        "id": 5229,
+        "logprob": -2.7988281,
+        "special": false,
+        "text": " failed"
+      },
+      {
+        "id": 29901,
+        "logprob": -0.91259766,
+        "special": false,
+        "text": ":"
+      },
+      {
+        "id": 853,
+        "logprob": -2.8496094,
+        "special": false,
+        "text": " Un"
+      },
+      {
+        "id": 23765,
+        "logprob": -1.1894531,
+        "special": false,
+        "text": "supported"
+      },
+      {
+        "id": 4714,
+        "logprob": -1.5917969,
+        "special": false,
+        "text": " browser"
+      },
+      {
+        "id": 29892,
+        "logprob": -0.34765625,
+        "special": false,
+        "text": ","
+      },
+      {
+        "id": 1873,
+        "logprob": -1.2695312,
+        "special": false,
+        "text": " version"
+      },
+      {
+        "id": 470,
+        "logprob": -0.25170898,
+        "special": false,
+        "text": " or"
+      },
+      {
+        "id": 7481,
+        "logprob": -0.21411133,
+        "special": false,
+        "text": " platform"
+      },
+      {
+        "id": 13,
+        "logprob": -1.1162109,
+        "special": false,
+        "text": "\n"
+      }
+    ],
+    "top_tokens": null
+  },
+  "generated_text": " failed: Unsupported browser, version or platform\n"
+}
diff --git a/integration-tests/models/__snapshots__/test_flash_llama_marlin_24/test_flash_llama_marlin24_all_params.json b/integration-tests/models/__snapshots__/test_flash_llama_marlin_24/test_flash_llama_marlin24_all_params.json
new file mode 100644
index 00000000..58cacb80
--- /dev/null
+++ b/integration-tests/models/__snapshots__/test_flash_llama_marlin_24/test_flash_llama_marlin24_all_params.json
@@ -0,0 +1,89 @@
+{
+  "details": {
+    "best_of_sequences": null,
+    "finish_reason": "length",
+    "generated_tokens": 10,
+    "prefill": [
+      {
+        "id": 1,
+        "logprob": null,
+        "text": "<s>"
+      },
+      {
+        "id": 4321,
+        "logprob": -9.0859375,
+        "text": "Test"
+      },
+      {
+        "id": 2009,
+        "logprob": -16.359375,
+        "text": "request"
+      }
+    ],
+    "seed": 0,
+    "tokens": [
+      {
+        "id": 5229,
+        "logprob": -0.6645508,
+        "special": false,
+        "text": " failed"
+      },
+      {
+        "id": 29901,
+        "logprob": 0.0,
+        "special": false,
+        "text": ":"
+      },
+      {
+        "id": 6527,
+        "logprob": -2.2324219,
+        "special": false,
+        "text": " Could"
+      },
+      {
+        "id": 451,
+        "logprob": 0.0,
+        "special": false,
+        "text": " not"
+      },
+      {
+        "id": 6088,
+        "logprob": -1.6074219,
+        "special": false,
+        "text": " parse"
+      },
+      {
+        "id": 1243,
+        "logprob": -1.6298828,
+        "special": false,
+        "text": " test"
+      },
+      {
+        "id": 1206,
+        "logprob": -0.72558594,
+        "special": false,
+        "text": " case"
+      },
+      {
+        "id": 1024,
+        "logprob": -0.40429688,
+        "special": false,
+        "text": " name"
+      },
+      {
+        "id": 515,
+        "logprob": 0.0,
+        "special": false,
+        "text": " from"
+      },
+      {
+        "id": 525,
+        "logprob": -1.2519531,
+        "special": false,
+        "text": " '"
+      }
+    ],
+    "top_tokens": null
+  },
+  "generated_text": "Test request failed: Could not parse test case name from '"
+}
diff --git a/integration-tests/models/__snapshots__/test_flash_llama_marlin_24/test_flash_llama_marlin24_load.json b/integration-tests/models/__snapshots__/test_flash_llama_marlin_24/test_flash_llama_marlin24_load.json
new file mode 100644
index 00000000..96a40fa4
--- /dev/null
+++ b/integration-tests/models/__snapshots__/test_flash_llama_marlin_24/test_flash_llama_marlin24_load.json
@@ -0,0 +1,358 @@
+[
+  {
+    "details": {
+      "best_of_sequences": null,
+      "finish_reason": "length",
+      "generated_tokens": 10,
+      "prefill": [
+        {
+          "id": 1,
+          "logprob": null,
+          "text": "<s>"
+        },
+        {
+          "id": 4321,
+          "logprob": -9.0859375,
+          "text": "Test"
+        },
+        {
+          "id": 2009,
+          "logprob": -16.359375,
+          "text": "request"
+        }
+      ],
+      "seed": null,
+      "tokens": [
+        {
+          "id": 5229,
+          "logprob": -2.7988281,
+          "special": false,
+          "text": " failed"
+        },
+        {
+          "id": 29901,
+          "logprob": -0.91259766,
+          "special": false,
+          "text": ":"
+        },
+        {
+          "id": 853,
+          "logprob": -2.8496094,
+          "special": false,
+          "text": " Un"
+        },
+        {
+          "id": 23765,
+          "logprob": -1.1894531,
+          "special": false,
+          "text": "supported"
+        },
+        {
+          "id": 4714,
+          "logprob": -1.5917969,
+          "special": false,
+          "text": " browser"
+        },
+        {
+          "id": 29892,
+          "logprob": -0.34765625,
+          "special": false,
+          "text": ","
+        },
+        {
+          "id": 1873,
+          "logprob": -1.2695312,
+          "special": false,
+          "text": " version"
+        },
+        {
+          "id": 470,
+          "logprob": -0.25170898,
+          "special": false,
+          "text": " or"
+        },
+        {
+          "id": 7481,
+          "logprob": -0.21411133,
+          "special": false,
+          "text": " platform"
+        },
+        {
+          "id": 13,
+          "logprob": -1.1162109,
+          "special": false,
+          "text": "\n"
+        }
+      ],
+      "top_tokens": null
+    },
+    "generated_text": " failed: Unsupported browser, version or platform\n"
+  },
+  {
+    "details": {
+      "best_of_sequences": null,
+      "finish_reason": "length",
+      "generated_tokens": 10,
+      "prefill": [
+        {
+          "id": 1,
+          "logprob": null,
+          "text": "<s>"
+        },
+        {
+          "id": 4321,
+          "logprob": -9.0859375,
+          "text": "Test"
+        },
+        {
+          "id": 2009,
+          "logprob": -16.359375,
+          "text": "request"
+        }
+      ],
+      "seed": null,
+      "tokens": [
+        {
+          "id": 5229,
+          "logprob": -2.7988281,
+          "special": false,
+          "text": " failed"
+        },
+        {
+          "id": 29901,
+          "logprob": -0.91259766,
+          "special": false,
+          "text": ":"
+        },
+        {
+          "id": 853,
+          "logprob": -2.8496094,
+          "special": false,
+          "text": " Un"
+        },
+        {
+          "id": 23765,
+          "logprob": -1.1894531,
+          "special": false,
+          "text": "supported"
+        },
+        {
+          "id": 4714,
+          "logprob": -1.5917969,
+          "special": false,
+          "text": " browser"
+        },
+        {
+          "id": 29892,
+          "logprob": -0.34765625,
+          "special": false,
+          "text": ","
+        },
+        {
+          "id": 1873,
+          "logprob": -1.2695312,
+          "special": false,
+          "text": " version"
+        },
+        {
+          "id": 470,
+          "logprob": -0.25170898,
+          "special": false,
+          "text": " or"
+        },
+        {
+          "id": 7481,
+          "logprob": -0.21411133,
+          "special": false,
+          "text": " platform"
+        },
+        {
+          "id": 13,
+          "logprob": -1.1162109,
+          "special": false,
+          "text": "\n"
+        }
+      ],
+      "top_tokens": null
+    },
+    "generated_text": " failed: Unsupported browser, version or platform\n"
+  },
+  {
+    "details": {
+      "best_of_sequences": null,
+      "finish_reason": "length",
+      "generated_tokens": 10,
+      "prefill": [
+        {
+          "id": 1,
+          "logprob": null,
+          "text": "<s>"
+        },
+        {
+          "id": 4321,
+          "logprob": -9.0859375,
+          "text": "Test"
+        },
+        {
+          "id": 2009,
+          "logprob": -16.359375,
+          "text": "request"
+        }
+      ],
+      "seed": null,
+      "tokens": [
+        {
+          "id": 5229,
+          "logprob": -2.7988281,
+          "special": false,
+          "text": " failed"
+        },
+        {
+          "id": 29901,
+          "logprob": -0.91259766,
+          "special": false,
+          "text": ":"
+        },
+        {
+          "id": 853,
+          "logprob": -2.8496094,
+          "special": false,
+          "text": " Un"
+        },
+        {
+          "id": 23765,
+          "logprob": -1.1894531,
+          "special": false,
+          "text": "supported"
+        },
+        {
+          "id": 4714,
+          "logprob": -1.5917969,
+          "special": false,
+          "text": " browser"
+        },
+        {
+          "id": 29892,
+          "logprob": -0.34765625,
+          "special": false,
+          "text": ","
+        },
+        {
+          "id": 1873,
+          "logprob": -1.2695312,
+          "special": false,
+          "text": " version"
+        },
+        {
+          "id": 470,
+          "logprob": -0.25170898,
+          "special": false,
+          "text": " or"
+        },
+        {
+          "id": 7481,
+          "logprob": -0.21411133,
+          "special": false,
+          "text": " platform"
+        },
+        {
+          "id": 13,
+          "logprob": -1.1162109,
+          "special": false,
+          "text": "\n"
+        }
+      ],
+      "top_tokens": null
+    },
+    "generated_text": " failed: Unsupported browser, version or platform\n"
+  },
+  {
+    "details": {
+      "best_of_sequences": null,
+      "finish_reason": "length",
+      "generated_tokens": 10,
+      "prefill": [
+        {
+          "id": 1,
+          "logprob": null,
+          "text": "<s>"
+        },
+        {
+          "id": 4321,
+          "logprob": -9.0859375,
+          "text": "Test"
+        },
+        {
+          "id": 2009,
+          "logprob": -16.359375,
+          "text": "request"
+        }
+      ],
+      "seed": null,
+      "tokens": [
+        {
+          "id": 5229,
+          "logprob": -2.7988281,
+          "special": false,
+          "text": " failed"
+        },
+        {
+          "id": 29901,
+          "logprob": -0.91259766,
+          "special": false,
+          "text": ":"
+        },
+        {
+          "id": 853,
+          "logprob": -2.8496094,
+          "special": false,
+          "text": " Un"
+        },
+        {
+          "id": 23765,
+          "logprob": -1.1894531,
+          "special": false,
+          "text": "supported"
+        },
+        {
+          "id": 4714,
+          "logprob": -1.5917969,
+          "special": false,
+          "text": " browser"
+        },
+        {
+          "id": 29892,
+          "logprob": -0.34765625,
+          "special": false,
+          "text": ","
+        },
+        {
+          "id": 1873,
+          "logprob": -1.2695312,
+          "special": false,
+          "text": " version"
+        },
+        {
+          "id": 470,
+          "logprob": -0.25170898,
+          "special": false,
+          "text": " or"
+        },
+        {
+          "id": 7481,
+          "logprob": -0.21411133,
+          "special": false,
+          "text": " platform"
+        },
+        {
+          "id": 13,
+          "logprob": -1.1162109,
+          "special": false,
+          "text": "\n"
+        }
+      ],
+      "top_tokens": null
+    },
+    "generated_text": " failed: Unsupported browser, version or platform\n"
+  }
+]
diff --git a/integration-tests/models/__snapshots__/test_flash_llama_prefix/test_flash_llama_load.json b/integration-tests/models/__snapshots__/test_flash_llama_prefix/test_flash_llama_load.json
new file mode 100644
index 00000000..b7c6177b
--- /dev/null
+++ b/integration-tests/models/__snapshots__/test_flash_llama_prefix/test_flash_llama_load.json
@@ -0,0 +1,2550 @@
+[
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Jeff Walker's Product Launch Formula is a comprehensive system",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085326,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 69,
+      "total_tokens": 79
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Here are three key indicators to determine if a customer",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085331,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 52,
+      "total_tokens": 62
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "You can use the `String.format()` method in",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085326,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 97,
+      "total_tokens": 107
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "In a realm of binary mysticism, we find",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085331,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 126,
+      "total_tokens": 136
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "The `dummy` variable is being used to consume",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085336,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 305,
+      "total_tokens": 315
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "You can add multiple new columns in Power Query (",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085331,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 51,
+      "total_tokens": 61
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "There are many exciting new technologies emerging across various fields",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085336,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 52,
+      "total_tokens": 62
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Poly Ether Ether Ketone (PEEK) is",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085326,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 40,
+      "total_tokens": 50
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Here's a technical overview of a referral system similar",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085326,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 85,
+      "total_tokens": 95
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Here's an example of how you can add an",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085336,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 45,
+      "total_tokens": 55
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "I'd be happy to help with Java. What",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085326,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 43,
+      "total_tokens": 53
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "I can help you plan a road trip from Pune",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085326,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 82,
+      "total_tokens": 92
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "I'd be happy to explain more about a topic",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085331,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 38,
+      "total_tokens": 48
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "I'd be happy to help you brainstorm and provide",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085326,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 47,
+      "total_tokens": 57
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Implementing a Minesweeper algorithm using algebraic",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085326,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 54,
+      "total_tokens": 64
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "There are several issues with the provided code:\n\n1",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085331,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 375,
+      "total_tokens": 385
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "stop",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": ";)",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085330,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 2,
+      "prompt_tokens": 105,
+      "total_tokens": 107
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "As I delved into the world of high-st",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085331,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 2097,
+      "total_tokens": 2107
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "/u/CruxHub: Hi, I'm",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085336,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 2614,
+      "total_tokens": 2624
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "To simulate a conversation between Alice and /u/C",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085331,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 1070,
+      "total_tokens": 1080
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Alice: Hey /u/CruxHub,",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085336,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 1847,
+      "total_tokens": 1857
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Alice: Hi /u/CruxHub,",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085331,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 1849,
+      "total_tokens": 1859
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "/u/CruxHub: Hey Alice, I",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085336,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 1004,
+      "total_tokens": 1014
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "/u/CruxHub: Hey Alice, I",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085331,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 1100,
+      "total_tokens": 1110
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "/u/CruxHub: Hey Alice, I",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085336,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 1044,
+      "total_tokens": 1054
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "The Dogme approach and the Lexical Approach are",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085336,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 54,
+      "total_tokens": 64
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Implementing a netfilter in Linux with a Rust",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085336,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 48,
+      "total_tokens": 58
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Damage to the Ulnar nerve can cause numb",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085331,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 56,
+      "total_tokens": 66
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "The Space Shuttle's Reaction Control System (RCS",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085331,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 50,
+      "total_tokens": 60
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "I can provide you with a basic Python script that",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085326,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 65,
+      "total_tokens": 75
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Farming meat has several negative impacts on the environment",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085331,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 43,
+      "total_tokens": 53
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "The photograph filter you're referring to is called \"",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085326,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 51,
+      "total_tokens": 61
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Here's a sample geological database structure with some example",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085326,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 59,
+      "total_tokens": 69
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "**Web Marketing: A Simplified Explanation**\n\nWeb",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085326,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 45,
+      "total_tokens": 55
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Here's a rewritten and improved version of the story",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085336,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 447,
+      "total_tokens": 457
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Here are the questions rewritten in a more conversational",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085331,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 168,
+      "total_tokens": 178
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "**Learning Progress: 0%**\n\n| Topic",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085331,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 216,
+      "total_tokens": 226
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "I couldn't find any information on a person named",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085326,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 44,
+      "total_tokens": 54
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Here's a list of the largest outdoor retailers in",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085326,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 43,
+      "total_tokens": 53
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "To create a WordPress shortcode that includes Facebook SDK code",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085326,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 49,
+      "total_tokens": 59
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "The sentence is mostly grammatically correct, but there",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085326,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 78,
+      "total_tokens": 88
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "I'd be happy to engage in a debate with",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085326,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 59,
+      "total_tokens": 69
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "I'd love to hear about your business. As",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085336,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 64,
+      "total_tokens": 74
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "I'll wait for your request to proceed with part",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085331,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 2410,
+      "total_tokens": 2420
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "The final part of the Day Sculpting program emphasizes",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085336,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 2699,
+      "total_tokens": 2709
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "**Analysis of the Coming of Age Story Archetype",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085336,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 349,
+      "total_tokens": 359
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "The Apostle John is one of the most prominent figures",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085326,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 49,
+      "total_tokens": 59
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "To build a Google Places autocomplete feature on Jetpack",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085336,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 427,
+      "total_tokens": 437
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "The information provided does not mention the captain's name",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085331,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 169,
+      "total_tokens": 179
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "The metaverse is a shared, immersive and interactive",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085336,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 39,
+      "total_tokens": 49
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Here are some ideas for a series of articles for",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085326,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 50,
+      "total_tokens": 60
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "\"Purim Palooza Alert: \n\nTo",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085331,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 78,
+      "total_tokens": 88
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "**Summary of the paper in 10 points:",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085331,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 2022,
+      "total_tokens": 2032
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "You'll provide three pieces of text, and then",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085336,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 58,
+      "total_tokens": 68
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "I'm ready to proceed with text 3.",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085336,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 1650,
+      "total_tokens": 1660
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "I'm ready to answer questions on Text 1",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085326,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 1116,
+      "total_tokens": 1126
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "This is a Solidity contract written in the older",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085331,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 334,
+      "total_tokens": 344
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "**Speech Recognition and Synthesis using Python**\n\nTo",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085326,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 84,
+      "total_tokens": 94
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "I'd be happy to help you discuss a paper",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085336,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 42,
+      "total_tokens": 52
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "To handle the given utterance, we can use",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085336,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 375,
+      "total_tokens": 385
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "**Subscription Services Template:**\n\n**Title:** Virtual",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085331,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 443,
+      "total_tokens": 453
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Hello. How can I assist you today?",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085331,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 36,
+      "total_tokens": 46
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Differentiating yourself from other Etsy shops is crucial to",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085326,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 102,
+      "total_tokens": 112
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "To become a Licensed Marriage and Family Therapist (",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085326,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 53,
+      "total_tokens": 63
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "**What is Quantum Computing?**\n\nQuantum computing",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085326,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 42,
+      "total_tokens": 52
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Aquí te dejo 40 opciones de nombres",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085326,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 108,
+      "total_tokens": 118
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Deposition is a geological process that involves the transportation",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085331,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 38,
+      "total_tokens": 48
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Here are some good e-governance initiatives in",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085336,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 55,
+      "total_tokens": 65
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Here's a simple Python program that accepts a command",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085326,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 56,
+      "total_tokens": 66
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Imagine you're playing with a toy box. You",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085326,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 47,
+      "total_tokens": 57
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Here's an example of a question they might ask",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085326,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 66,
+      "total_tokens": 76
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Arduino Uno adalah sebuah papan mikrokontrol",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085326,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 38,
+      "total_tokens": 48
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "To edit an array that is within an object,",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085326,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 42,
+      "total_tokens": 52
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Microsoft ENTRA (Enterprise Mobility + Security) is",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085336,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 56,
+      "total_tokens": 66
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "To calculate the difference in interest paid between a simple",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085331,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 69,
+      "total_tokens": 79
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Yes, you can use Spring State Machine and Spring",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085326,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 49,
+      "total_tokens": 59
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "The issue lies in the fact that the `meta",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085326,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 142,
+      "total_tokens": 152
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Here are some effective marketing tactics for local small businesses",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085336,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 46,
+      "total_tokens": 56
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "The French Revolution, which lasted from 1789",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085331,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 41,
+      "total_tokens": 51
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "**Roles of a Network Driver:**\n\nA network",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085336,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 65,
+      "total_tokens": 75
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Yes, I'm familiar with the SAS (Stat",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085331,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 44,
+      "total_tokens": 54
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Using relays to control 12V solen",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085336,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 60,
+      "total_tokens": 70
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "You can use the following Python code to achieve this",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085331,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 55,
+      "total_tokens": 65
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Here are some prompts for viral comics:\n\n1.",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085336,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 336,
+      "total_tokens": 346
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "To simplify and make the comic funnier, consider",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085336,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 301,
+      "total_tokens": 311
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Here's a rewritten version of the 4-panel",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085331,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 282,
+      "total_tokens": 292
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Subject: Request for E-Waste Collection and Computer",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085331,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 110,
+      "total_tokens": 120
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "In the context of conference calls, the state you",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085331,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 84,
+      "total_tokens": 94
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "I can provide a general classification of companies based on",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085326,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 56,
+      "total_tokens": 66
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Here are some user stories that describe the concept in",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085336,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 44,
+      "total_tokens": 54
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "You can check your Python version by running the following",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085326,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 39,
+      "total_tokens": 49
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "**Scenario:**\n\n15-year-old Black youth,",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085336,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 473,
+      "total_tokens": 483
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "As a Demand Generation Manager for a B2B",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085336,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 50,
+      "total_tokens": 60
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "The error is due to a typo in your code",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085336,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 369,
+      "total_tokens": 379
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "고등교육의 필요성에 관한 영어 에",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085326,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 72,
+      "total_tokens": 82
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Here's a simple C# program that uses the",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085326,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 51,
+      "total_tokens": 61
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "The error message \"connection refused\" indicates that the",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085331,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 85,
+      "total_tokens": 95
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "To load an image, you can use various methods",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726085326,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 41,
+      "total_tokens": 51
+    }
+  }
+]
diff --git a/integration-tests/models/__snapshots__/test_flash_llama_prefix_flashdecoding/test_flash_llama_flashdecoding.json b/integration-tests/models/__snapshots__/test_flash_llama_prefix_flashdecoding/test_flash_llama_flashdecoding.json
new file mode 100644
index 00000000..51247ba4
--- /dev/null
+++ b/integration-tests/models/__snapshots__/test_flash_llama_prefix_flashdecoding/test_flash_llama_flashdecoding.json
@@ -0,0 +1,2550 @@
+[
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Jeff Walker's Product Launch Formula is a comprehensive system",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243286,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 69,
+      "total_tokens": 79
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Here are three key indicators to determine if a customer",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243278,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 52,
+      "total_tokens": 62
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "You can use the `String.format()` method in",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243277,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 97,
+      "total_tokens": 107
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "In a realm of binary mysticism, we find",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243278,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 126,
+      "total_tokens": 136
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "The `dummy` variable is being used to consume",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243283,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 305,
+      "total_tokens": 315
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "You can add multiple new columns in Power Query (",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243286,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 51,
+      "total_tokens": 61
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "There are many exciting new technologies emerging across various fields",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243277,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 52,
+      "total_tokens": 62
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Poly Ether Ether Ketone (PEEK) is",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243278,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 40,
+      "total_tokens": 50
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Here's a technical overview of a referral system similar",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243277,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 85,
+      "total_tokens": 95
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Here's an example of how you can add an",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243284,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 45,
+      "total_tokens": 55
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "I'd be happy to help with Java. What",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243283,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 43,
+      "total_tokens": 53
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "I can help you plan a road trip from Pune",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243277,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 82,
+      "total_tokens": 92
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "I'd be happy to explain more about a topic",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243284,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 38,
+      "total_tokens": 48
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "I'd be happy to help you brainstorm and provide",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243278,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 47,
+      "total_tokens": 57
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Implementing a Minesweeper algorithm using algebraic",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243278,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 54,
+      "total_tokens": 64
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "There are several issues with the provided code:\n\n1",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243284,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 375,
+      "total_tokens": 385
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "stop",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": ";)",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243277,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 2,
+      "prompt_tokens": 105,
+      "total_tokens": 107
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "As I delved into the world of high-st",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243277,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 2097,
+      "total_tokens": 2107
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "/u/CruxHub: Hi, I'm",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243283,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 2614,
+      "total_tokens": 2624
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "To simulate a conversation between Alice and /u/C",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243278,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 1070,
+      "total_tokens": 1080
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Alice: Hey /u/CruxHub,",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243283,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 1847,
+      "total_tokens": 1857
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Alice: Hi /u/CruxHub,",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243286,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 1849,
+      "total_tokens": 1859
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "/u/CruxHub: Hey Alice, I",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243278,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 1004,
+      "total_tokens": 1014
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "/u/CruxHub: Hey Alice, I",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243286,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 1100,
+      "total_tokens": 1110
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "/u/CruxHub: Hey Alice, I",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243278,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 1044,
+      "total_tokens": 1054
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "The Dogme approach and the Lexical Approach are",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243278,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 54,
+      "total_tokens": 64
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Implementing a netfilter in Linux with a Rust",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243278,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 48,
+      "total_tokens": 58
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Damage to the Ulnar nerve can cause numb",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243277,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 56,
+      "total_tokens": 66
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "The Space Shuttle's Reaction Control System (RCS",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243277,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 50,
+      "total_tokens": 60
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "I can provide you with a basic Python script that",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243278,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 65,
+      "total_tokens": 75
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Farming meat has several negative impacts on the environment",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243277,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 43,
+      "total_tokens": 53
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "The photograph filter you're referring to is called \"",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243277,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 51,
+      "total_tokens": 61
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Here's a sample geological database structure with some example",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243283,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 59,
+      "total_tokens": 69
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "**Web Marketing: A Simplified Explanation**\n\nWeb",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243278,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 45,
+      "total_tokens": 55
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Here's a rewritten and improved version of the story",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243278,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 447,
+      "total_tokens": 457
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Here are the questions rewritten in a more conversational",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243278,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 168,
+      "total_tokens": 178
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "**Learning Progress: 0%**\n\n| Topic",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243284,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 216,
+      "total_tokens": 226
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "I couldn't find any information on a person named",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243284,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 44,
+      "total_tokens": 54
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Here's a list of the largest outdoor retailers in",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243286,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 43,
+      "total_tokens": 53
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "To create a WordPress shortcode that includes Facebook SDK code",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243284,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 49,
+      "total_tokens": 59
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "The sentence is mostly grammatically correct, but there",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243277,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 78,
+      "total_tokens": 88
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "I'd be happy to engage in a debate with",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243284,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 59,
+      "total_tokens": 69
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "I'd love to hear about your business. As",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243284,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 64,
+      "total_tokens": 74
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "I'll wait for your request to proceed with part",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243286,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 2410,
+      "total_tokens": 2420
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "The final part of the Day Sculpting program emphasizes",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243284,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 2699,
+      "total_tokens": 2709
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "**Analysis of the Coming of Age Story Archetype",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243278,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 349,
+      "total_tokens": 359
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "The Apostle John is one of the most prominent figures",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243277,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 49,
+      "total_tokens": 59
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "To build a Google Places autocomplete feature on Jetpack",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243277,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 427,
+      "total_tokens": 437
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "The information provided does not mention the captain's name",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243283,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 169,
+      "total_tokens": 179
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "The metaverse is a shared, immersive and interactive",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243277,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 39,
+      "total_tokens": 49
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Here are some ideas for a series of articles for",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243278,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 50,
+      "total_tokens": 60
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "\"Purim Palooza Alert: \n\nTo",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243284,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 78,
+      "total_tokens": 88
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "**Summary of the paper in 10 points:",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243286,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 2022,
+      "total_tokens": 2032
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "You'll provide three pieces of text, and then",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243283,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 58,
+      "total_tokens": 68
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "I'm ready to proceed with text 3.",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243284,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 1650,
+      "total_tokens": 1660
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "I'm ready to answer questions on Text 1",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243286,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 1116,
+      "total_tokens": 1126
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "This is a Solidity contract written in the older",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243278,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 334,
+      "total_tokens": 344
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "**Speech Recognition and Synthesis using Python**\n\nTo",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243283,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 84,
+      "total_tokens": 94
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "I'd be happy to help you discuss a paper",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243286,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 42,
+      "total_tokens": 52
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "To handle the given utterance, we can use",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243277,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 375,
+      "total_tokens": 385
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "**Subscription Services Template:**\n\n**Title:** Virtual",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243286,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 443,
+      "total_tokens": 453
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Hello. How can I assist you today?",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243278,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 36,
+      "total_tokens": 46
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Differentiating yourself from other Etsy shops is crucial to",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243283,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 102,
+      "total_tokens": 112
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "To become a Licensed Marriage and Family Therapist (",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243284,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 53,
+      "total_tokens": 63
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "**What is Quantum Computing?**\n\nQuantum computing",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243278,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 42,
+      "total_tokens": 52
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Aquí te dejo 40 opciones de nombres",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243283,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 108,
+      "total_tokens": 118
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Deposition is a geological process that involves the transportation",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243278,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 38,
+      "total_tokens": 48
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Here are some good e-governance initiatives in",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243284,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 55,
+      "total_tokens": 65
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Here's a simple Python program that accepts a command",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243278,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 56,
+      "total_tokens": 66
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Imagine you're playing with a toy box. You",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243284,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 47,
+      "total_tokens": 57
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Here's an example of a question they might ask",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243278,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 66,
+      "total_tokens": 76
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Arduino Uno adalah sebuah papan mikrokontrol",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243283,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 38,
+      "total_tokens": 48
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "To edit an array that is within an object,",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243278,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 42,
+      "total_tokens": 52
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Microsoft ENTRA (Enterprise Mobility + Security) is",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243286,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 56,
+      "total_tokens": 66
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "To calculate the difference in interest paid between a simple",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243277,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 69,
+      "total_tokens": 79
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Yes, you can use Spring State Machine and Spring",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243278,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 49,
+      "total_tokens": 59
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "The issue lies in the fact that the `meta",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243278,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 142,
+      "total_tokens": 152
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Here are some effective marketing tactics for local small businesses",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243286,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 46,
+      "total_tokens": 56
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "The French Revolution, which lasted from 1789",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243283,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 41,
+      "total_tokens": 51
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "**Roles of a Network Driver:**\n\nA network",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243278,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 65,
+      "total_tokens": 75
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Yes, I'm familiar with the SAS (Stat",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243277,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 44,
+      "total_tokens": 54
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Using relays to control 12V solen",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243284,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 60,
+      "total_tokens": 70
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "You can use the following Python code to achieve this",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243278,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 55,
+      "total_tokens": 65
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Here are some prompts for viral comics:\n\n1.",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243284,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 336,
+      "total_tokens": 346
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "To simplify and make the comic funnier, consider",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243278,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 301,
+      "total_tokens": 311
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Here's a rewritten version of the 4-panel",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243278,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 282,
+      "total_tokens": 292
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Subject: Request for E-Waste Collection and Computer",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243284,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 110,
+      "total_tokens": 120
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "In the context of conference calls, the state you",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243277,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 84,
+      "total_tokens": 94
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "I can provide a general classification of companies based on",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243284,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 56,
+      "total_tokens": 66
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Here are some user stories that describe the concept in",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243283,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 44,
+      "total_tokens": 54
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "You can check your Python version by running the following",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243284,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 39,
+      "total_tokens": 49
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "**Scenario:**\n\n15-year-old Black youth,",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243284,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 473,
+      "total_tokens": 483
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "As a Demand Generation Manager for a B2B",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243277,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 50,
+      "total_tokens": 60
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "The error is due to a typo in your code",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243277,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 369,
+      "total_tokens": 379
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "고등교육의 필요성에 관한 영어 에",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243286,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 72,
+      "total_tokens": 82
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "Here's a simple C# program that uses the",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243283,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 51,
+      "total_tokens": 61
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "The error message \"connection refused\" indicates that the",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243277,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 85,
+      "total_tokens": 95
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "To load an image, you can use various methods",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1726243284,
+    "id": "",
+    "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.2.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 41,
+      "total_tokens": 51
+    }
+  }
+]
diff --git a/integration-tests/models/__snapshots__/test_flash_mixtral/test_flash_mixtral.json b/integration-tests/models/__snapshots__/test_flash_mixtral/test_flash_mixtral.json
new file mode 100644
index 00000000..56419967
--- /dev/null
+++ b/integration-tests/models/__snapshots__/test_flash_mixtral/test_flash_mixtral.json
@@ -0,0 +1,114 @@
+{
+  "details": {
+    "best_of_sequences": null,
+    "finish_reason": "length",
+    "generated_tokens": 10,
+    "prefill": [
+      {
+        "id": 1,
+        "logprob": null,
+        "text": "<s>"
+      },
+      {
+        "id": 1824,
+        "logprob": -6.1445312,
+        "text": "What"
+      },
+      {
+        "id": 349,
+        "logprob": -1.4648438,
+        "text": "is"
+      },
+      {
+        "id": 21135,
+        "logprob": -13.6875,
+        "text": "gradient"
+      },
+      {
+        "id": 24871,
+        "logprob": -1.6005859,
+        "text": "descent"
+      },
+      {
+        "id": 28804,
+        "logprob": -0.39526367,
+        "text": "?"
+      },
+      {
+        "id": 13,
+        "logprob": -0.640625,
+        "text": "\n"
+      },
+      {
+        "id": 13,
+        "logprob": -0.18774414,
+        "text": "\n"
+      }
+    ],
+    "seed": null,
+    "tokens": [
+      {
+        "id": 20910,
+        "logprob": -0.96484375,
+        "special": false,
+        "text": "Grad"
+      },
+      {
+        "id": 722,
+        "logprob": -0.003168106,
+        "special": false,
+        "text": "ient"
+      },
+      {
+        "id": 24871,
+        "logprob": -0.16540527,
+        "special": false,
+        "text": " descent"
+      },
+      {
+        "id": 349,
+        "logprob": -0.08886719,
+        "special": false,
+        "text": " is"
+      },
+      {
+        "id": 396,
+        "logprob": -0.75878906,
+        "special": false,
+        "text": " an"
+      },
+      {
+        "id": 18586,
+        "logprob": -0.5703125,
+        "special": false,
+        "text": " optimization"
+      },
+      {
+        "id": 9464,
+        "logprob": -0.11242676,
+        "special": false,
+        "text": " algorithm"
+      },
+      {
+        "id": 1307,
+        "logprob": -0.7939453,
+        "special": false,
+        "text": " used"
+      },
+      {
+        "id": 298,
+        "logprob": -0.17102051,
+        "special": false,
+        "text": " to"
+      },
+      {
+        "id": 26518,
+        "logprob": -0.34326172,
+        "special": false,
+        "text": " minimize"
+      }
+    ],
+    "top_tokens": null
+  },
+  "generated_text": "Gradient descent is an optimization algorithm used to minimize"
+}
diff --git a/integration-tests/models/__snapshots__/test_flash_mixtral/test_flash_mixtral_all_params.json b/integration-tests/models/__snapshots__/test_flash_mixtral/test_flash_mixtral_all_params.json
new file mode 100644
index 00000000..e25086c6
--- /dev/null
+++ b/integration-tests/models/__snapshots__/test_flash_mixtral/test_flash_mixtral_all_params.json
@@ -0,0 +1,99 @@
+{
+  "details": {
+    "best_of_sequences": null,
+    "finish_reason": "length",
+    "generated_tokens": 10,
+    "prefill": [
+      {
+        "id": 1,
+        "logprob": null,
+        "text": "<s>"
+      },
+      {
+        "id": 24871,
+        "logprob": -17.234375,
+        "text": "descent"
+      },
+      {
+        "id": 28804,
+        "logprob": -7.4375,
+        "text": "?"
+      },
+      {
+        "id": 13,
+        "logprob": -0.8046875,
+        "text": "\n"
+      },
+      {
+        "id": 13,
+        "logprob": -0.33032227,
+        "text": "\n"
+      }
+    ],
+    "seed": 0,
+    "tokens": [
+      {
+        "id": 1313,
+        "logprob": -2.3613281,
+        "special": false,
+        "text": "It"
+      },
+      {
+        "id": 3969,
+        "logprob": -0.7285156,
+        "special": false,
+        "text": " seems"
+      },
+      {
+        "id": 298,
+        "logprob": -1.3466797,
+        "special": false,
+        "text": " to"
+      },
+      {
+        "id": 528,
+        "logprob": 0.0,
+        "special": false,
+        "text": " me"
+      },
+      {
+        "id": 28725,
+        "logprob": -1.6757812,
+        "special": false,
+        "text": ","
+      },
+      {
+        "id": 369,
+        "logprob": 0.0,
+        "special": false,
+        "text": " that"
+      },
+      {
+        "id": 513,
+        "logprob": -1.1269531,
+        "special": false,
+        "text": " if"
+      },
+      {
+        "id": 368,
+        "logprob": 0.0,
+        "special": false,
+        "text": " you"
+      },
+      {
+        "id": 28742,
+        "logprob": -2.4921875,
+        "special": false,
+        "text": "'"
+      },
+      {
+        "id": 267,
+        "logprob": 0.0,
+        "special": false,
+        "text": "re"
+      }
+    ],
+    "top_tokens": null
+  },
+  "generated_text": "What is gradient descent?\n\nIt seems to me, that if you're"
+}
diff --git a/integration-tests/models/__snapshots__/test_flash_mixtral/test_flash_mixtral_load.json b/integration-tests/models/__snapshots__/test_flash_mixtral/test_flash_mixtral_load.json
new file mode 100644
index 00000000..55056cfd
--- /dev/null
+++ b/integration-tests/models/__snapshots__/test_flash_mixtral/test_flash_mixtral_load.json
@@ -0,0 +1,458 @@
+[
+  {
+    "details": {
+      "best_of_sequences": null,
+      "finish_reason": "length",
+      "generated_tokens": 10,
+      "prefill": [
+        {
+          "id": 1,
+          "logprob": null,
+          "text": "<s>"
+        },
+        {
+          "id": 1824,
+          "logprob": -6.1445312,
+          "text": "What"
+        },
+        {
+          "id": 349,
+          "logprob": -1.4648438,
+          "text": "is"
+        },
+        {
+          "id": 21135,
+          "logprob": -13.6875,
+          "text": "gradient"
+        },
+        {
+          "id": 24871,
+          "logprob": -1.6005859,
+          "text": "descent"
+        },
+        {
+          "id": 28804,
+          "logprob": -0.39526367,
+          "text": "?"
+        },
+        {
+          "id": 13,
+          "logprob": -0.640625,
+          "text": "\n"
+        },
+        {
+          "id": 13,
+          "logprob": -0.18774414,
+          "text": "\n"
+        }
+      ],
+      "seed": null,
+      "tokens": [
+        {
+          "id": 20910,
+          "logprob": -0.96484375,
+          "special": false,
+          "text": "Grad"
+        },
+        {
+          "id": 722,
+          "logprob": -0.003168106,
+          "special": false,
+          "text": "ient"
+        },
+        {
+          "id": 24871,
+          "logprob": -0.16369629,
+          "special": false,
+          "text": " descent"
+        },
+        {
+          "id": 349,
+          "logprob": -0.0881958,
+          "special": false,
+          "text": " is"
+        },
+        {
+          "id": 396,
+          "logprob": -0.76708984,
+          "special": false,
+          "text": " an"
+        },
+        {
+          "id": 18586,
+          "logprob": -0.57373047,
+          "special": false,
+          "text": " optimization"
+        },
+        {
+          "id": 9464,
+          "logprob": -0.11291504,
+          "special": false,
+          "text": " algorithm"
+        },
+        {
+          "id": 1307,
+          "logprob": -0.79589844,
+          "special": false,
+          "text": " used"
+        },
+        {
+          "id": 298,
+          "logprob": -0.1694336,
+          "special": false,
+          "text": " to"
+        },
+        {
+          "id": 26518,
+          "logprob": -0.34350586,
+          "special": false,
+          "text": " minimize"
+        }
+      ],
+      "top_tokens": null
+    },
+    "generated_text": "Gradient descent is an optimization algorithm used to minimize"
+  },
+  {
+    "details": {
+      "best_of_sequences": null,
+      "finish_reason": "length",
+      "generated_tokens": 10,
+      "prefill": [
+        {
+          "id": 1,
+          "logprob": null,
+          "text": "<s>"
+        },
+        {
+          "id": 1824,
+          "logprob": -6.1445312,
+          "text": "What"
+        },
+        {
+          "id": 349,
+          "logprob": -1.4677734,
+          "text": "is"
+        },
+        {
+          "id": 21135,
+          "logprob": -13.6875,
+          "text": "gradient"
+        },
+        {
+          "id": 24871,
+          "logprob": -1.6015625,
+          "text": "descent"
+        },
+        {
+          "id": 28804,
+          "logprob": -0.39453125,
+          "text": "?"
+        },
+        {
+          "id": 13,
+          "logprob": -0.6435547,
+          "text": "\n"
+        },
+        {
+          "id": 13,
+          "logprob": -0.18713379,
+          "text": "\n"
+        }
+      ],
+      "seed": null,
+      "tokens": [
+        {
+          "id": 20910,
+          "logprob": -0.9628906,
+          "special": false,
+          "text": "Grad"
+        },
+        {
+          "id": 722,
+          "logprob": -0.0032176971,
+          "special": false,
+          "text": "ient"
+        },
+        {
+          "id": 24871,
+          "logprob": -0.16540527,
+          "special": false,
+          "text": " descent"
+        },
+        {
+          "id": 349,
+          "logprob": -0.08898926,
+          "special": false,
+          "text": " is"
+        },
+        {
+          "id": 396,
+          "logprob": -0.765625,
+          "special": false,
+          "text": " an"
+        },
+        {
+          "id": 18586,
+          "logprob": -0.5708008,
+          "special": false,
+          "text": " optimization"
+        },
+        {
+          "id": 9464,
+          "logprob": -0.11401367,
+          "special": false,
+          "text": " algorithm"
+        },
+        {
+          "id": 1307,
+          "logprob": -0.7963867,
+          "special": false,
+          "text": " used"
+        },
+        {
+          "id": 298,
+          "logprob": -0.17028809,
+          "special": false,
+          "text": " to"
+        },
+        {
+          "id": 26518,
+          "logprob": -0.34326172,
+          "special": false,
+          "text": " minimize"
+        }
+      ],
+      "top_tokens": null
+    },
+    "generated_text": "Gradient descent is an optimization algorithm used to minimize"
+  },
+  {
+    "details": {
+      "best_of_sequences": null,
+      "finish_reason": "length",
+      "generated_tokens": 10,
+      "prefill": [
+        {
+          "id": 1,
+          "logprob": null,
+          "text": "<s>"
+        },
+        {
+          "id": 1824,
+          "logprob": -6.140625,
+          "text": "What"
+        },
+        {
+          "id": 349,
+          "logprob": -1.4658203,
+          "text": "is"
+        },
+        {
+          "id": 21135,
+          "logprob": -13.6796875,
+          "text": "gradient"
+        },
+        {
+          "id": 24871,
+          "logprob": -1.5898438,
+          "text": "descent"
+        },
+        {
+          "id": 28804,
+          "logprob": -0.3955078,
+          "text": "?"
+        },
+        {
+          "id": 13,
+          "logprob": -0.64501953,
+          "text": "\n"
+        },
+        {
+          "id": 13,
+          "logprob": -0.18493652,
+          "text": "\n"
+        }
+      ],
+      "seed": null,
+      "tokens": [
+        {
+          "id": 20910,
+          "logprob": -0.9580078,
+          "special": false,
+          "text": "Grad"
+        },
+        {
+          "id": 722,
+          "logprob": -0.0032176971,
+          "special": false,
+          "text": "ient"
+        },
+        {
+          "id": 24871,
+          "logprob": -0.16552734,
+          "special": false,
+          "text": " descent"
+        },
+        {
+          "id": 349,
+          "logprob": -0.08874512,
+          "special": false,
+          "text": " is"
+        },
+        {
+          "id": 396,
+          "logprob": -0.75878906,
+          "special": false,
+          "text": " an"
+        },
+        {
+          "id": 18586,
+          "logprob": -0.5703125,
+          "special": false,
+          "text": " optimization"
+        },
+        {
+          "id": 9464,
+          "logprob": -0.11236572,
+          "special": false,
+          "text": " algorithm"
+        },
+        {
+          "id": 1307,
+          "logprob": -0.79541016,
+          "special": false,
+          "text": " used"
+        },
+        {
+          "id": 298,
+          "logprob": -0.17102051,
+          "special": false,
+          "text": " to"
+        },
+        {
+          "id": 26518,
+          "logprob": -0.34326172,
+          "special": false,
+          "text": " minimize"
+        }
+      ],
+      "top_tokens": null
+    },
+    "generated_text": "Gradient descent is an optimization algorithm used to minimize"
+  },
+  {
+    "details": {
+      "best_of_sequences": null,
+      "finish_reason": "length",
+      "generated_tokens": 10,
+      "prefill": [
+        {
+          "id": 1,
+          "logprob": null,
+          "text": "<s>"
+        },
+        {
+          "id": 1824,
+          "logprob": -6.1328125,
+          "text": "What"
+        },
+        {
+          "id": 349,
+          "logprob": -1.4658203,
+          "text": "is"
+        },
+        {
+          "id": 21135,
+          "logprob": -13.6796875,
+          "text": "gradient"
+        },
+        {
+          "id": 24871,
+          "logprob": -1.5947266,
+          "text": "descent"
+        },
+        {
+          "id": 28804,
+          "logprob": -0.39648438,
+          "text": "?"
+        },
+        {
+          "id": 13,
+          "logprob": -0.6464844,
+          "text": "\n"
+        },
+        {
+          "id": 13,
+          "logprob": -0.18688965,
+          "text": "\n"
+        }
+      ],
+      "seed": null,
+      "tokens": [
+        {
+          "id": 20910,
+          "logprob": -0.9609375,
+          "special": false,
+          "text": "Grad"
+        },
+        {
+          "id": 722,
+          "logprob": -0.003168106,
+          "special": false,
+          "text": "ient"
+        },
+        {
+          "id": 24871,
+          "logprob": -0.16601562,
+          "special": false,
+          "text": " descent"
+        },
+        {
+          "id": 349,
+          "logprob": -0.088134766,
+          "special": false,
+          "text": " is"
+        },
+        {
+          "id": 396,
+          "logprob": -0.7597656,
+          "special": false,
+          "text": " an"
+        },
+        {
+          "id": 18586,
+          "logprob": -0.5708008,
+          "special": false,
+          "text": " optimization"
+        },
+        {
+          "id": 9464,
+          "logprob": -0.11291504,
+          "special": false,
+          "text": " algorithm"
+        },
+        {
+          "id": 1307,
+          "logprob": -0.7944336,
+          "special": false,
+          "text": " used"
+        },
+        {
+          "id": 298,
+          "logprob": -0.17102051,
+          "special": false,
+          "text": " to"
+        },
+        {
+          "id": 26518,
+          "logprob": -0.34399414,
+          "special": false,
+          "text": " minimize"
+        }
+      ],
+      "top_tokens": null
+    },
+    "generated_text": "Gradient descent is an optimization algorithm used to minimize"
+  }
+]
diff --git a/integration-tests/models/__snapshots__/test_flash_mixtral_gptq/test_flash_mixtral_gptq.json b/integration-tests/models/__snapshots__/test_flash_mixtral_gptq/test_flash_mixtral_gptq.json
new file mode 100644
index 00000000..993bdadd
--- /dev/null
+++ b/integration-tests/models/__snapshots__/test_flash_mixtral_gptq/test_flash_mixtral_gptq.json
@@ -0,0 +1,89 @@
+{
+  "details": {
+    "best_of_sequences": null,
+    "finish_reason": "length",
+    "generated_tokens": 10,
+    "prefill": [
+      {
+        "id": 1,
+        "logprob": null,
+        "text": "<s>"
+      },
+      {
+        "id": 3735,
+        "logprob": -11.0078125,
+        "text": "Test"
+      },
+      {
+        "id": 2159,
+        "logprob": -13.59375,
+        "text": "request"
+      }
+    ],
+    "seed": null,
+    "tokens": [
+      {
+        "id": 13,
+        "logprob": -1.7089844,
+        "special": false,
+        "text": "\n"
+      },
+      {
+        "id": 13,
+        "logprob": -0.68847656,
+        "special": false,
+        "text": "\n"
+      },
+      {
+        "id": 28771,
+        "logprob": -1.9394531,
+        "special": false,
+        "text": "#"
+      },
+      {
+        "id": 3735,
+        "logprob": -2.8808594,
+        "special": false,
+        "text": " Test"
+      },
+      {
+        "id": 2159,
+        "logprob": -0.37280273,
+        "special": false,
+        "text": " request"
+      },
+      {
+        "id": 13,
+        "logprob": -0.26098633,
+        "special": false,
+        "text": "\n"
+      },
+      {
+        "id": 13,
+        "logprob": -0.0017137527,
+        "special": false,
+        "text": "\n"
+      },
+      {
+        "id": 1064,
+        "logprob": -2.2695312,
+        "special": false,
+        "text": "##"
+      },
+      {
+        "id": 3735,
+        "logprob": -1.9238281,
+        "special": false,
+        "text": " Test"
+      },
+      {
+        "id": 2159,
+        "logprob": -0.48828125,
+        "special": false,
+        "text": " request"
+      }
+    ],
+    "top_tokens": null
+  },
+  "generated_text": "\n\n# Test request\n\n## Test request"
+}
diff --git a/integration-tests/models/__snapshots__/test_flash_mixtral_gptq/test_flash_mixtral_gptq_all_params.json b/integration-tests/models/__snapshots__/test_flash_mixtral_gptq/test_flash_mixtral_gptq_all_params.json
new file mode 100644
index 00000000..94411eef
--- /dev/null
+++ b/integration-tests/models/__snapshots__/test_flash_mixtral_gptq/test_flash_mixtral_gptq_all_params.json
@@ -0,0 +1,89 @@
+{
+  "details": {
+    "best_of_sequences": null,
+    "finish_reason": "length",
+    "generated_tokens": 10,
+    "prefill": [
+      {
+        "id": 1,
+        "logprob": null,
+        "text": "<s>"
+      },
+      {
+        "id": 3735,
+        "logprob": -11.0078125,
+        "text": "Test"
+      },
+      {
+        "id": 2159,
+        "logprob": -13.59375,
+        "text": "request"
+      }
+    ],
+    "seed": 0,
+    "tokens": [
+      {
+        "id": 13,
+        "logprob": -0.34838867,
+        "special": false,
+        "text": "\n"
+      },
+      {
+        "id": 13940,
+        "logprob": -0.38916016,
+        "special": false,
+        "text": "``"
+      },
+      {
+        "id": 28832,
+        "logprob": 0.0,
+        "special": false,
+        "text": "`"
+      },
+      {
+        "id": 3371,
+        "logprob": -1.2529297,
+        "special": false,
+        "text": "json"
+      },
+      {
+        "id": 13,
+        "logprob": 0.0,
+        "special": false,
+        "text": "\n"
+      },
+      {
+        "id": 28751,
+        "logprob": 0.0,
+        "special": false,
+        "text": "{"
+      },
+      {
+        "id": 13,
+        "logprob": 0.0,
+        "special": false,
+        "text": "\n"
+      },
+      {
+        "id": 2287,
+        "logprob": 0.0,
+        "special": false,
+        "text": "   "
+      },
+      {
+        "id": 345,
+        "logprob": 0.0,
+        "special": false,
+        "text": " \""
+      },
+      {
+        "id": 3134,
+        "logprob": -0.640625,
+        "special": false,
+        "text": "request"
+      }
+    ],
+    "top_tokens": null
+  },
+  "generated_text": "Test request\n```json\n{\n    \"request"
+}
diff --git a/integration-tests/models/__snapshots__/test_flash_mixtral_gptq/test_flash_mixtral_gptq_load.json b/integration-tests/models/__snapshots__/test_flash_mixtral_gptq/test_flash_mixtral_gptq_load.json
new file mode 100644
index 00000000..19e306a3
--- /dev/null
+++ b/integration-tests/models/__snapshots__/test_flash_mixtral_gptq/test_flash_mixtral_gptq_load.json
@@ -0,0 +1,358 @@
+[
+  {
+    "details": {
+      "best_of_sequences": null,
+      "finish_reason": "length",
+      "generated_tokens": 10,
+      "prefill": [
+        {
+          "id": 1,
+          "logprob": null,
+          "text": "<s>"
+        },
+        {
+          "id": 3735,
+          "logprob": -11.0078125,
+          "text": "Test"
+        },
+        {
+          "id": 2159,
+          "logprob": -13.59375,
+          "text": "request"
+        }
+      ],
+      "seed": null,
+      "tokens": [
+        {
+          "id": 13,
+          "logprob": -1.7089844,
+          "special": false,
+          "text": "\n"
+        },
+        {
+          "id": 13,
+          "logprob": -0.68847656,
+          "special": false,
+          "text": "\n"
+        },
+        {
+          "id": 28771,
+          "logprob": -1.9394531,
+          "special": false,
+          "text": "#"
+        },
+        {
+          "id": 3735,
+          "logprob": -2.8828125,
+          "special": false,
+          "text": " Test"
+        },
+        {
+          "id": 2159,
+          "logprob": -0.37329102,
+          "special": false,
+          "text": " request"
+        },
+        {
+          "id": 13,
+          "logprob": -0.2602539,
+          "special": false,
+          "text": "\n"
+        },
+        {
+          "id": 13,
+          "logprob": -0.0017185211,
+          "special": false,
+          "text": "\n"
+        },
+        {
+          "id": 1064,
+          "logprob": -2.2753906,
+          "special": false,
+          "text": "##"
+        },
+        {
+          "id": 3735,
+          "logprob": -1.9316406,
+          "special": false,
+          "text": " Test"
+        },
+        {
+          "id": 2159,
+          "logprob": -0.48217773,
+          "special": false,
+          "text": " request"
+        }
+      ],
+      "top_tokens": null
+    },
+    "generated_text": "\n\n# Test request\n\n## Test request"
+  },
+  {
+    "details": {
+      "best_of_sequences": null,
+      "finish_reason": "length",
+      "generated_tokens": 10,
+      "prefill": [
+        {
+          "id": 1,
+          "logprob": null,
+          "text": "<s>"
+        },
+        {
+          "id": 3735,
+          "logprob": -11.0078125,
+          "text": "Test"
+        },
+        {
+          "id": 2159,
+          "logprob": -13.59375,
+          "text": "request"
+        }
+      ],
+      "seed": null,
+      "tokens": [
+        {
+          "id": 13,
+          "logprob": -1.7089844,
+          "special": false,
+          "text": "\n"
+        },
+        {
+          "id": 13,
+          "logprob": -0.68847656,
+          "special": false,
+          "text": "\n"
+        },
+        {
+          "id": 28771,
+          "logprob": -1.9394531,
+          "special": false,
+          "text": "#"
+        },
+        {
+          "id": 3735,
+          "logprob": -2.8828125,
+          "special": false,
+          "text": " Test"
+        },
+        {
+          "id": 2159,
+          "logprob": -0.37329102,
+          "special": false,
+          "text": " request"
+        },
+        {
+          "id": 13,
+          "logprob": -0.2602539,
+          "special": false,
+          "text": "\n"
+        },
+        {
+          "id": 13,
+          "logprob": -0.0017185211,
+          "special": false,
+          "text": "\n"
+        },
+        {
+          "id": 1064,
+          "logprob": -2.2753906,
+          "special": false,
+          "text": "##"
+        },
+        {
+          "id": 3735,
+          "logprob": -1.9316406,
+          "special": false,
+          "text": " Test"
+        },
+        {
+          "id": 2159,
+          "logprob": -0.48217773,
+          "special": false,
+          "text": " request"
+        }
+      ],
+      "top_tokens": null
+    },
+    "generated_text": "\n\n# Test request\n\n## Test request"
+  },
+  {
+    "details": {
+      "best_of_sequences": null,
+      "finish_reason": "length",
+      "generated_tokens": 10,
+      "prefill": [
+        {
+          "id": 1,
+          "logprob": null,
+          "text": "<s>"
+        },
+        {
+          "id": 3735,
+          "logprob": -11.0078125,
+          "text": "Test"
+        },
+        {
+          "id": 2159,
+          "logprob": -13.59375,
+          "text": "request"
+        }
+      ],
+      "seed": null,
+      "tokens": [
+        {
+          "id": 13,
+          "logprob": -1.7089844,
+          "special": false,
+          "text": "\n"
+        },
+        {
+          "id": 13,
+          "logprob": -0.68847656,
+          "special": false,
+          "text": "\n"
+        },
+        {
+          "id": 28771,
+          "logprob": -1.9394531,
+          "special": false,
+          "text": "#"
+        },
+        {
+          "id": 3735,
+          "logprob": -2.8828125,
+          "special": false,
+          "text": " Test"
+        },
+        {
+          "id": 2159,
+          "logprob": -0.37329102,
+          "special": false,
+          "text": " request"
+        },
+        {
+          "id": 13,
+          "logprob": -0.2602539,
+          "special": false,
+          "text": "\n"
+        },
+        {
+          "id": 13,
+          "logprob": -0.0017185211,
+          "special": false,
+          "text": "\n"
+        },
+        {
+          "id": 1064,
+          "logprob": -2.2753906,
+          "special": false,
+          "text": "##"
+        },
+        {
+          "id": 3735,
+          "logprob": -1.9316406,
+          "special": false,
+          "text": " Test"
+        },
+        {
+          "id": 2159,
+          "logprob": -0.48217773,
+          "special": false,
+          "text": " request"
+        }
+      ],
+      "top_tokens": null
+    },
+    "generated_text": "\n\n# Test request\n\n## Test request"
+  },
+  {
+    "details": {
+      "best_of_sequences": null,
+      "finish_reason": "length",
+      "generated_tokens": 10,
+      "prefill": [
+        {
+          "id": 1,
+          "logprob": null,
+          "text": "<s>"
+        },
+        {
+          "id": 3735,
+          "logprob": -11.0078125,
+          "text": "Test"
+        },
+        {
+          "id": 2159,
+          "logprob": -13.59375,
+          "text": "request"
+        }
+      ],
+      "seed": null,
+      "tokens": [
+        {
+          "id": 13,
+          "logprob": -1.7089844,
+          "special": false,
+          "text": "\n"
+        },
+        {
+          "id": 13,
+          "logprob": -0.68847656,
+          "special": false,
+          "text": "\n"
+        },
+        {
+          "id": 28771,
+          "logprob": -1.9394531,
+          "special": false,
+          "text": "#"
+        },
+        {
+          "id": 3735,
+          "logprob": -2.8828125,
+          "special": false,
+          "text": " Test"
+        },
+        {
+          "id": 2159,
+          "logprob": -0.37329102,
+          "special": false,
+          "text": " request"
+        },
+        {
+          "id": 13,
+          "logprob": -0.2602539,
+          "special": false,
+          "text": "\n"
+        },
+        {
+          "id": 13,
+          "logprob": -0.0017185211,
+          "special": false,
+          "text": "\n"
+        },
+        {
+          "id": 1064,
+          "logprob": -2.2753906,
+          "special": false,
+          "text": "##"
+        },
+        {
+          "id": 3735,
+          "logprob": -1.9316406,
+          "special": false,
+          "text": " Test"
+        },
+        {
+          "id": 2159,
+          "logprob": -0.48217773,
+          "special": false,
+          "text": " request"
+        }
+      ],
+      "top_tokens": null
+    },
+    "generated_text": "\n\n# Test request\n\n## Test request"
+  }
+]
diff --git a/integration-tests/models/__snapshots__/test_flash_pali_gemma/test_flash_pali_gemma.json b/integration-tests/models/__snapshots__/test_flash_pali_gemma/test_flash_pali_gemma.json
index 037e0b16..0096f807 100644
--- a/integration-tests/models/__snapshots__/test_flash_pali_gemma/test_flash_pali_gemma.json
+++ b/integration-tests/models/__snapshots__/test_flash_pali_gemma/test_flash_pali_gemma.json
@@ -8,13 +8,13 @@
     "tokens": [
       {
         "id": 54901,
-        "logprob": -0.72753906,
+        "logprob": -0.84765625,
         "special": false,
         "text": "beach"
       },
       {
         "id": 1,
-        "logprob": -0.011009216,
+        "logprob": -0.008666992,
         "special": true,
         "text": "<eos>"
       }
diff --git a/integration-tests/models/__snapshots__/test_flash_pali_gemma/test_flash_pali_gemma_two_images.json b/integration-tests/models/__snapshots__/test_flash_pali_gemma/test_flash_pali_gemma_two_images.json
new file mode 100644
index 00000000..e3b5575c
--- /dev/null
+++ b/integration-tests/models/__snapshots__/test_flash_pali_gemma/test_flash_pali_gemma_two_images.json
@@ -0,0 +1,61 @@
+{
+  "details": {
+    "best_of_sequences": null,
+    "finish_reason": "eos_token",
+    "generated_tokens": 8,
+    "prefill": [],
+    "seed": null,
+    "tokens": [
+      {
+        "id": 2502,
+        "logprob": -1.7890625,
+        "special": false,
+        "text": "image"
+      },
+      {
+        "id": 2196,
+        "logprob": -0.53125,
+        "special": false,
+        "text": " result"
+      },
+      {
+        "id": 604,
+        "logprob": -0.0077209473,
+        "special": false,
+        "text": " for"
+      },
+      {
+        "id": 12254,
+        "logprob": -1.703125,
+        "special": false,
+        "text": " chicken"
+      },
+      {
+        "id": 611,
+        "logprob": -0.21582031,
+        "special": false,
+        "text": " on"
+      },
+      {
+        "id": 573,
+        "logprob": -0.734375,
+        "special": false,
+        "text": " the"
+      },
+      {
+        "id": 8318,
+        "logprob": -0.026000977,
+        "special": false,
+        "text": " beach"
+      },
+      {
+        "id": 1,
+        "logprob": -0.2109375,
+        "special": true,
+        "text": "<eos>"
+      }
+    ],
+    "top_tokens": null
+  },
+  "generated_text": "image result for chicken on the beach"
+}
diff --git a/integration-tests/models/__snapshots__/test_flash_phi/test_flash_phi_all_params.json b/integration-tests/models/__snapshots__/test_flash_phi/test_flash_phi_all_params.json
index 221ff13d..b407ef71 100644
--- a/integration-tests/models/__snapshots__/test_flash_phi/test_flash_phi_all_params.json
+++ b/integration-tests/models/__snapshots__/test_flash_phi/test_flash_phi_all_params.json
@@ -19,25 +19,25 @@
     "tokens": [
       {
         "id": 284,
-        "logprob": -0.19421387,
+        "logprob": -0.28955078,
         "special": false,
         "text": " to"
       },
       {
         "id": 3758,
-        "logprob": -0.62597656,
+        "logprob": -0.7739258,
         "special": false,
         "text": " send"
       },
       {
         "id": 1366,
-        "logprob": -0.87060547,
+        "logprob": -0.85253906,
         "special": false,
         "text": " data"
       },
       {
         "id": 625,
-        "logprob": -0.88427734,
+        "logprob": -0.8984375,
         "special": false,
         "text": " over"
       },
@@ -49,7 +49,7 @@
       },
       {
         "id": 3127,
-        "logprob": -1.9462891,
+        "logprob": -1.9404297,
         "special": false,
         "text": " network"
       }
diff --git a/integration-tests/models/__snapshots__/test_flash_phi35_moe/test_flash_phi35_moe.json b/integration-tests/models/__snapshots__/test_flash_phi35_moe/test_flash_phi35_moe.json
new file mode 100644
index 00000000..0d6dca31
--- /dev/null
+++ b/integration-tests/models/__snapshots__/test_flash_phi35_moe/test_flash_phi35_moe.json
@@ -0,0 +1,109 @@
+{
+  "details": {
+    "best_of_sequences": null,
+    "finish_reason": "length",
+    "generated_tokens": 10,
+    "prefill": [
+      {
+        "id": 1724,
+        "logprob": null,
+        "text": "What"
+      },
+      {
+        "id": 338,
+        "logprob": -0.7133789,
+        "text": "is"
+      },
+      {
+        "id": 16030,
+        "logprob": -13.9296875,
+        "text": "gradient"
+      },
+      {
+        "id": 26815,
+        "logprob": -0.048919678,
+        "text": "descent"
+      },
+      {
+        "id": 29973,
+        "logprob": -3.0078125,
+        "text": "?"
+      },
+      {
+        "id": 13,
+        "logprob": -2.8105469,
+        "text": "\n"
+      },
+      {
+        "id": 13,
+        "logprob": -0.84521484,
+        "text": "\n"
+      }
+    ],
+    "seed": null,
+    "tokens": [
+      {
+        "id": 25584,
+        "logprob": -0.017028809,
+        "special": false,
+        "text": "Grad"
+      },
+      {
+        "id": 993,
+        "logprob": -0.0027313232,
+        "special": false,
+        "text": "ient"
+      },
+      {
+        "id": 26815,
+        "logprob": -0.023254395,
+        "special": false,
+        "text": " descent"
+      },
+      {
+        "id": 338,
+        "logprob": -2.0623207e-05,
+        "special": false,
+        "text": " is"
+      },
+      {
+        "id": 263,
+        "logprob": -0.5361328,
+        "special": false,
+        "text": " a"
+      },
+      {
+        "id": 937,
+        "logprob": -0.17578125,
+        "special": false,
+        "text": " first"
+      },
+      {
+        "id": 29899,
+        "logprob": 0.0,
+        "special": false,
+        "text": "-"
+      },
+      {
+        "id": 2098,
+        "logprob": -0.00011539459,
+        "special": false,
+        "text": "order"
+      },
+      {
+        "id": 13883,
+        "logprob": -0.47436523,
+        "special": false,
+        "text": " optimization"
+      },
+      {
+        "id": 5687,
+        "logprob": -0.00027680397,
+        "special": false,
+        "text": " algorithm"
+      }
+    ],
+    "top_tokens": null
+  },
+  "generated_text": "Gradient descent is a first-order optimization algorithm"
+}
diff --git a/integration-tests/models/__snapshots__/test_flash_phi35_moe/test_flash_phi35_moe_all_params.json b/integration-tests/models/__snapshots__/test_flash_phi35_moe/test_flash_phi35_moe_all_params.json
new file mode 100644
index 00000000..38b80335
--- /dev/null
+++ b/integration-tests/models/__snapshots__/test_flash_phi35_moe/test_flash_phi35_moe_all_params.json
@@ -0,0 +1,99 @@
+{
+  "details": {
+    "best_of_sequences": null,
+    "finish_reason": "length",
+    "generated_tokens": 10,
+    "prefill": [
+      {
+        "id": 16030,
+        "logprob": null,
+        "text": "gradient"
+      },
+      {
+        "id": 26815,
+        "logprob": -6.4960938,
+        "text": "descent"
+      },
+      {
+        "id": 29973,
+        "logprob": -5.1484375,
+        "text": "?"
+      },
+      {
+        "id": 13,
+        "logprob": -4.0351562,
+        "text": "\n"
+      },
+      {
+        "id": 13,
+        "logprob": -5.2265625,
+        "text": "\n"
+      }
+    ],
+    "seed": 0,
+    "tokens": [
+      {
+        "id": 10994,
+        "logprob": -1.1542969,
+        "special": false,
+        "text": "Hello"
+      },
+      {
+        "id": 29991,
+        "logprob": 0.0,
+        "special": false,
+        "text": "!"
+      },
+      {
+        "id": 739,
+        "logprob": 0.0,
+        "special": false,
+        "text": " It"
+      },
+      {
+        "id": 2444,
+        "logprob": -0.42260742,
+        "special": false,
+        "text": " seems"
+      },
+      {
+        "id": 366,
+        "logprob": 0.0,
+        "special": false,
+        "text": " you"
+      },
+      {
+        "id": 29915,
+        "logprob": 0.0,
+        "special": false,
+        "text": "'"
+      },
+      {
+        "id": 276,
+        "logprob": -0.9838867,
+        "special": false,
+        "text": "re"
+      },
+      {
+        "id": 3211,
+        "logprob": 0.0,
+        "special": false,
+        "text": " address"
+      },
+      {
+        "id": 292,
+        "logprob": 0.0,
+        "special": false,
+        "text": "ing"
+      },
+      {
+        "id": 263,
+        "logprob": -0.15124512,
+        "special": false,
+        "text": " a"
+      }
+    ],
+    "top_tokens": null
+  },
+  "generated_text": "What is gradient descent?\n\nHello! It seems you're addressing a"
+}
diff --git a/integration-tests/models/__snapshots__/test_flash_phi35_moe/test_flash_phi35_moe_load.json b/integration-tests/models/__snapshots__/test_flash_phi35_moe/test_flash_phi35_moe_load.json
new file mode 100644
index 00000000..f1f81152
--- /dev/null
+++ b/integration-tests/models/__snapshots__/test_flash_phi35_moe/test_flash_phi35_moe_load.json
@@ -0,0 +1,438 @@
+[
+  {
+    "details": {
+      "best_of_sequences": null,
+      "finish_reason": "length",
+      "generated_tokens": 10,
+      "prefill": [
+        {
+          "id": 1724,
+          "logprob": null,
+          "text": "What"
+        },
+        {
+          "id": 338,
+          "logprob": -0.7133789,
+          "text": "is"
+        },
+        {
+          "id": 16030,
+          "logprob": -13.9296875,
+          "text": "gradient"
+        },
+        {
+          "id": 26815,
+          "logprob": -0.048919678,
+          "text": "descent"
+        },
+        {
+          "id": 29973,
+          "logprob": -3.0078125,
+          "text": "?"
+        },
+        {
+          "id": 13,
+          "logprob": -2.8105469,
+          "text": "\n"
+        },
+        {
+          "id": 13,
+          "logprob": -0.84521484,
+          "text": "\n"
+        }
+      ],
+      "seed": null,
+      "tokens": [
+        {
+          "id": 25584,
+          "logprob": -0.017028809,
+          "special": false,
+          "text": "Grad"
+        },
+        {
+          "id": 993,
+          "logprob": -0.0028476715,
+          "special": false,
+          "text": "ient"
+        },
+        {
+          "id": 26815,
+          "logprob": -0.023971558,
+          "special": false,
+          "text": " descent"
+        },
+        {
+          "id": 338,
+          "logprob": -2.0384789e-05,
+          "special": false,
+          "text": " is"
+        },
+        {
+          "id": 263,
+          "logprob": -0.5229492,
+          "special": false,
+          "text": " a"
+        },
+        {
+          "id": 937,
+          "logprob": -0.17602539,
+          "special": false,
+          "text": " first"
+        },
+        {
+          "id": 29899,
+          "logprob": 0.0,
+          "special": false,
+          "text": "-"
+        },
+        {
+          "id": 2098,
+          "logprob": -0.000116467476,
+          "special": false,
+          "text": "order"
+        },
+        {
+          "id": 13883,
+          "logprob": -0.47436523,
+          "special": false,
+          "text": " optimization"
+        },
+        {
+          "id": 5687,
+          "logprob": -0.00027871132,
+          "special": false,
+          "text": " algorithm"
+        }
+      ],
+      "top_tokens": null
+    },
+    "generated_text": "Gradient descent is a first-order optimization algorithm"
+  },
+  {
+    "details": {
+      "best_of_sequences": null,
+      "finish_reason": "length",
+      "generated_tokens": 10,
+      "prefill": [
+        {
+          "id": 1724,
+          "logprob": null,
+          "text": "What"
+        },
+        {
+          "id": 338,
+          "logprob": -0.7128906,
+          "text": "is"
+        },
+        {
+          "id": 16030,
+          "logprob": -13.9375,
+          "text": "gradient"
+        },
+        {
+          "id": 26815,
+          "logprob": -0.05053711,
+          "text": "descent"
+        },
+        {
+          "id": 29973,
+          "logprob": -3.0058594,
+          "text": "?"
+        },
+        {
+          "id": 13,
+          "logprob": -2.8242188,
+          "text": "\n"
+        },
+        {
+          "id": 13,
+          "logprob": -0.84521484,
+          "text": "\n"
+        }
+      ],
+      "seed": null,
+      "tokens": [
+        {
+          "id": 25584,
+          "logprob": -0.018859863,
+          "special": false,
+          "text": "Grad"
+        },
+        {
+          "id": 993,
+          "logprob": -0.002822876,
+          "special": false,
+          "text": "ient"
+        },
+        {
+          "id": 26815,
+          "logprob": -0.023254395,
+          "special": false,
+          "text": " descent"
+        },
+        {
+          "id": 338,
+          "logprob": -2.0384789e-05,
+          "special": false,
+          "text": " is"
+        },
+        {
+          "id": 263,
+          "logprob": -0.5229492,
+          "special": false,
+          "text": " a"
+        },
+        {
+          "id": 937,
+          "logprob": -0.17126465,
+          "special": false,
+          "text": " first"
+        },
+        {
+          "id": 29899,
+          "logprob": 0.0,
+          "special": false,
+          "text": "-"
+        },
+        {
+          "id": 2098,
+          "logprob": -0.0001155138,
+          "special": false,
+          "text": "order"
+        },
+        {
+          "id": 13883,
+          "logprob": -0.47436523,
+          "special": false,
+          "text": " optimization"
+        },
+        {
+          "id": 5687,
+          "logprob": -0.00027036667,
+          "special": false,
+          "text": " algorithm"
+        }
+      ],
+      "top_tokens": null
+    },
+    "generated_text": "Gradient descent is a first-order optimization algorithm"
+  },
+  {
+    "details": {
+      "best_of_sequences": null,
+      "finish_reason": "length",
+      "generated_tokens": 10,
+      "prefill": [
+        {
+          "id": 1724,
+          "logprob": null,
+          "text": "What"
+        },
+        {
+          "id": 338,
+          "logprob": -0.71484375,
+          "text": "is"
+        },
+        {
+          "id": 16030,
+          "logprob": -13.9375,
+          "text": "gradient"
+        },
+        {
+          "id": 26815,
+          "logprob": -0.049346924,
+          "text": "descent"
+        },
+        {
+          "id": 29973,
+          "logprob": -3.0078125,
+          "text": "?"
+        },
+        {
+          "id": 13,
+          "logprob": -2.8242188,
+          "text": "\n"
+        },
+        {
+          "id": 13,
+          "logprob": -0.86328125,
+          "text": "\n"
+        }
+      ],
+      "seed": null,
+      "tokens": [
+        {
+          "id": 25584,
+          "logprob": -0.017196655,
+          "special": false,
+          "text": "Grad"
+        },
+        {
+          "id": 993,
+          "logprob": -0.0028438568,
+          "special": false,
+          "text": "ient"
+        },
+        {
+          "id": 26815,
+          "logprob": -0.023254395,
+          "special": false,
+          "text": " descent"
+        },
+        {
+          "id": 338,
+          "logprob": -2.026558e-05,
+          "special": false,
+          "text": " is"
+        },
+        {
+          "id": 263,
+          "logprob": -0.5229492,
+          "special": false,
+          "text": " a"
+        },
+        {
+          "id": 937,
+          "logprob": -0.17602539,
+          "special": false,
+          "text": " first"
+        },
+        {
+          "id": 29899,
+          "logprob": 0.0,
+          "special": false,
+          "text": "-"
+        },
+        {
+          "id": 2098,
+          "logprob": -0.00011622906,
+          "special": false,
+          "text": "order"
+        },
+        {
+          "id": 13883,
+          "logprob": -0.48608398,
+          "special": false,
+          "text": " optimization"
+        },
+        {
+          "id": 5687,
+          "logprob": -0.00027894974,
+          "special": false,
+          "text": " algorithm"
+        }
+      ],
+      "top_tokens": null
+    },
+    "generated_text": "Gradient descent is a first-order optimization algorithm"
+  },
+  {
+    "details": {
+      "best_of_sequences": null,
+      "finish_reason": "length",
+      "generated_tokens": 10,
+      "prefill": [
+        {
+          "id": 1724,
+          "logprob": null,
+          "text": "What"
+        },
+        {
+          "id": 338,
+          "logprob": -0.7192383,
+          "text": "is"
+        },
+        {
+          "id": 16030,
+          "logprob": -13.9375,
+          "text": "gradient"
+        },
+        {
+          "id": 26815,
+          "logprob": -0.050445557,
+          "text": "descent"
+        },
+        {
+          "id": 29973,
+          "logprob": -3.0078125,
+          "text": "?"
+        },
+        {
+          "id": 13,
+          "logprob": -2.8242188,
+          "text": "\n"
+        },
+        {
+          "id": 13,
+          "logprob": -0.8276367,
+          "text": "\n"
+        }
+      ],
+      "seed": null,
+      "tokens": [
+        {
+          "id": 25584,
+          "logprob": -0.01727295,
+          "special": false,
+          "text": "Grad"
+        },
+        {
+          "id": 993,
+          "logprob": -0.0027542114,
+          "special": false,
+          "text": "ient"
+        },
+        {
+          "id": 26815,
+          "logprob": -0.023254395,
+          "special": false,
+          "text": " descent"
+        },
+        {
+          "id": 338,
+          "logprob": -2.0384789e-05,
+          "special": false,
+          "text": " is"
+        },
+        {
+          "id": 263,
+          "logprob": -0.5229492,
+          "special": false,
+          "text": " a"
+        },
+        {
+          "id": 937,
+          "logprob": -0.17126465,
+          "special": false,
+          "text": " first"
+        },
+        {
+          "id": 29899,
+          "logprob": 0.0,
+          "special": false,
+          "text": "-"
+        },
+        {
+          "id": 2098,
+          "logprob": -0.00011301041,
+          "special": false,
+          "text": "order"
+        },
+        {
+          "id": 13883,
+          "logprob": -0.48608398,
+          "special": false,
+          "text": " optimization"
+        },
+        {
+          "id": 5687,
+          "logprob": -0.00027894974,
+          "special": false,
+          "text": " algorithm"
+        }
+      ],
+      "top_tokens": null
+    },
+    "generated_text": "Gradient descent is a first-order optimization algorithm"
+  }
+]
diff --git a/integration-tests/models/__snapshots__/test_flash_starcoder/test_flash_starcoder_default_params.json b/integration-tests/models/__snapshots__/test_flash_starcoder/test_flash_starcoder_default_params.json
index 89e02c07..164e3cf2 100644
--- a/integration-tests/models/__snapshots__/test_flash_starcoder/test_flash_starcoder_default_params.json
+++ b/integration-tests/models/__snapshots__/test_flash_starcoder/test_flash_starcoder_default_params.json
@@ -11,17 +11,17 @@
       },
       {
         "id": 1459,
-        "logprob": -5.6328125,
+        "logprob": -5.625,
         "text": " print"
       },
       {
         "id": 81,
-        "logprob": -1.6035156,
+        "logprob": -1.6064453,
         "text": "_"
       },
       {
         "id": 7656,
-        "logprob": -5.9882812,
+        "logprob": -5.9921875,
         "text": "hello"
       }
     ],
@@ -29,7 +29,7 @@
     "tokens": [
       {
         "id": 2262,
-        "logprob": -0.042999268,
+        "logprob": -0.045715332,
         "special": false,
         "text": "():"
       },
@@ -59,7 +59,7 @@
       },
       {
         "id": 10896,
-        "logprob": -0.38549805,
+        "logprob": -0.3659668,
         "special": false,
         "text": " World"
       },
@@ -113,7 +113,7 @@
       },
       {
         "id": 426,
-        "logprob": 0.0,
+        "logprob": -0.051635742,
         "special": false,
         "text": "name"
       },
@@ -323,7 +323,7 @@
       },
       {
         "id": 313,
-        "logprob": -0.6328125,
+        "logprob": -0.6933594,
         "special": false,
         "text": " \""
       },
@@ -387,7 +387,8 @@
         "special": false,
         "text": " print"
       }
-    ]
+    ],
+    "top_tokens": null
   },
   "generated_text": "():\n    print(\"Hello World\")\n\ndef print_hello_name(name):\n    print(\"Hello \" + name)\n\ndef print_hello_name_age(name, age):\n    print(\"Hello \" + name + \" \" + str(age))\n\ndef print"
 }
diff --git a/integration-tests/models/__snapshots__/test_flash_starcoder2/test_flash_starcoder2_default_params.json b/integration-tests/models/__snapshots__/test_flash_starcoder2/test_flash_starcoder2_default_params.json
index 38117272..412b19b4 100644
--- a/integration-tests/models/__snapshots__/test_flash_starcoder2/test_flash_starcoder2_default_params.json
+++ b/integration-tests/models/__snapshots__/test_flash_starcoder2/test_flash_starcoder2_default_params.json
@@ -11,12 +11,12 @@
       },
       {
         "id": 1489,
-        "logprob": -5.2617188,
+        "logprob": -5.265625,
         "text": " print"
       },
       {
         "id": 100,
-        "logprob": -0.38476562,
+        "logprob": -0.38305664,
         "text": "_"
       },
       {
@@ -53,13 +53,13 @@
       },
       {
         "id": 8302,
-        "logprob": -0.28125,
+        "logprob": -0.26611328,
         "special": false,
         "text": "Hello"
       },
       {
         "id": 10914,
-        "logprob": -0.79248047,
+        "logprob": -0.7734375,
         "special": false,
         "text": " World"
       },
@@ -71,7 +71,7 @@
       },
       {
         "id": 222,
-        "logprob": -0.0619812,
+        "logprob": -0.054870605,
         "special": false,
         "text": "\n"
       },
@@ -83,7 +83,7 @@
       },
       {
         "id": 610,
-        "logprob": -0.4091797,
+        "logprob": -0.4152832,
         "special": false,
         "text": "def"
       },
@@ -113,7 +113,7 @@
       },
       {
         "id": 444,
-        "logprob": -0.21655273,
+        "logprob": -0.21618652,
         "special": false,
         "text": "name"
       },
@@ -160,16 +160,28 @@
         "text": "Hello"
       },
       {
-        "id": 332,
-        "logprob": -0.034698486,
+        "id": 925,
+        "logprob": -3.3476562,
         "special": false,
-        "text": " \""
+        "text": " %"
       },
       {
-        "id": 494,
+        "id": 120,
         "logprob": 0.0,
         "special": false,
-        "text": " +"
+        "text": "s"
+      },
+      {
+        "id": 11571,
+        "logprob": -0.08892822,
+        "special": false,
+        "text": "!\""
+      },
+      {
+        "id": 925,
+        "logprob": 0.0,
+        "special": false,
+        "text": " %"
       },
       {
         "id": 655,
@@ -178,22 +190,10 @@
         "text": " name"
       },
       {
-        "id": 494,
-        "logprob": -0.20141602,
-        "special": false,
-        "text": " +"
-      },
-      {
-        "id": 332,
+        "id": 46,
         "logprob": 0.0,
         "special": false,
-        "text": " \""
-      },
-      {
-        "id": 16013,
-        "logprob": 0.0,
-        "special": false,
-        "text": "!\")"
+        "text": ")"
       },
       {
         "id": 222,
@@ -251,7 +251,7 @@
       },
       {
         "id": 400,
-        "logprob": 0.0,
+        "logprob": -0.074279785,
         "special": false,
         "text": "age"
       },
@@ -310,34 +310,22 @@
         "text": "Hello"
       },
       {
-        "id": 332,
+        "id": 925,
         "logprob": 0.0,
         "special": false,
-        "text": " \""
+        "text": " %"
       },
       {
-        "id": 494,
+        "id": 120,
         "logprob": 0.0,
         "special": false,
-        "text": " +"
+        "text": "s"
       },
       {
-        "id": 655,
-        "logprob": 0.0,
+        "id": 49,
+        "logprob": -0.07891846,
         "special": false,
-        "text": " name"
-      },
-      {
-        "id": 494,
-        "logprob": 0.0,
-        "special": false,
-        "text": " +"
-      },
-      {
-        "id": 3021,
-        "logprob": -0.5761719,
-        "special": false,
-        "text": " \","
+        "text": ","
       },
       {
         "id": 863,
@@ -352,43 +340,55 @@
         "text": " are"
       },
       {
-        "id": 332,
+        "id": 925,
         "logprob": 0.0,
         "special": false,
-        "text": " \""
+        "text": " %"
       },
       {
-        "id": 494,
+        "id": 105,
         "logprob": 0.0,
         "special": false,
-        "text": " +"
+        "text": "d"
       },
       {
-        "id": 615,
+        "id": 11339,
         "logprob": 0.0,
         "special": false,
-        "text": " str"
+        "text": " years"
       },
       {
-        "id": 45,
+        "id": 3627,
         "logprob": 0.0,
         "special": false,
-        "text": "("
+        "text": " old"
       },
       {
-        "id": 400,
+        "id": 11571,
         "logprob": 0.0,
         "special": false,
-        "text": "age"
+        "text": "!\""
       },
       {
-        "id": 46,
+        "id": 925,
         "logprob": 0.0,
         "special": false,
-        "text": ")"
+        "text": " %"
+      },
+      {
+        "id": 327,
+        "logprob": 0.0,
+        "special": false,
+        "text": " ("
+      },
+      {
+        "id": 444,
+        "logprob": 0.0,
+        "special": false,
+        "text": "name"
       }
     ],
     "top_tokens": null
   },
-  "generated_text": "():\n    print(\"Hello World!\")\n\ndef print_hello_name(name):\n    print(\"Hello \" + name + \"!\")\n\ndef print_hello_name_age(name, age):\n    print(\"Hello \" + name + \", you are \" + str(age)"
+  "generated_text": "():\n    print(\"Hello World!\")\n\ndef print_hello_name(name):\n    print(\"Hello %s!\" % name)\n\ndef print_hello_name_age(name, age):\n    print(\"Hello %s, you are %d years old!\" % (name"
 }
diff --git a/integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq.json b/integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq.json
index 5e537bb7..26224118 100644
--- a/integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq.json
+++ b/integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq.json
@@ -1,8 +1,8 @@
 {
   "details": {
     "best_of_sequences": null,
-    "finish_reason": "length",
-    "generated_tokens": 20,
+    "finish_reason": "eos_token",
+    "generated_tokens": 2,
     "prefill": [
       {
         "id": 589,
@@ -11,57 +11,57 @@
       },
       {
         "id": 3226,
-        "logprob": -8.5859375,
+        "logprob": -8.9453125,
         "text": " ge"
       },
       {
         "id": 21017,
-        "logprob": -7.5859375,
+        "logprob": -8.8515625,
         "text": "ometric"
       },
       {
         "id": 81,
-        "logprob": -0.2668457,
+        "logprob": -0.21875,
         "text": "_"
       },
       {
         "id": 6009,
-        "logprob": -1.6416016,
+        "logprob": -1.2773438,
         "text": "mean"
       },
       {
         "id": 26,
-        "logprob": -0.22705078,
+        "logprob": -0.25195312,
         "text": "("
       },
       {
         "id": 62,
-        "logprob": -5.2304688,
+        "logprob": -4.8203125,
         "text": "L"
       },
       {
         "id": 44,
-        "logprob": -3.0976562,
+        "logprob": -3.7734375,
         "text": ":"
       },
       {
         "id": 1682,
-        "logprob": -1.1044922,
+        "logprob": -0.8310547,
         "text": " List"
       },
       {
         "id": 77,
-        "logprob": -0.14294434,
+        "logprob": -0.22766113,
         "text": "["
       },
       {
         "id": 1808,
-        "logprob": -0.32299805,
+        "logprob": -0.46240234,
         "text": "float"
       },
       {
         "id": 10794,
-        "logprob": -2.8164062,
+        "logprob": -3.0234375,
         "text": "]):"
       }
     ],
@@ -69,126 +69,18 @@
     "tokens": [
       {
         "id": 284,
-        "logprob": -0.1282959,
+        "logprob": -0.04626465,
         "special": false,
         "text": "\n   "
       },
       {
-        "id": 1524,
-        "logprob": -0.97998047,
-        "special": false,
-        "text": " \"\"\""
-      },
-      {
-        "id": 284,
-        "logprob": -0.7006836,
-        "special": false,
-        "text": "\n   "
-      },
-      {
-        "id": 14883,
-        "logprob": -2.1933594,
-        "special": false,
-        "text": " Calculate"
-      },
-      {
-        "id": 322,
-        "logprob": -0.2697754,
-        "special": false,
-        "text": " the"
-      },
-      {
-        "id": 3226,
-        "logprob": -0.0836792,
-        "special": false,
-        "text": " ge"
-      },
-      {
-        "id": 21017,
-        "logprob": -0.018737793,
-        "special": false,
-        "text": "ometric"
-      },
-      {
-        "id": 5651,
-        "logprob": -0.028640747,
-        "special": false,
-        "text": " mean"
-      },
-      {
-        "id": 432,
-        "logprob": -0.29467773,
-        "special": false,
-        "text": " of"
-      },
-      {
-        "id": 312,
-        "logprob": -0.31518555,
-        "special": false,
-        "text": " a"
-      },
-      {
-        "id": 1149,
-        "logprob": -0.20605469,
-        "special": false,
-        "text": " list"
-      },
-      {
-        "id": 432,
-        "logprob": -0.23254395,
-        "special": false,
-        "text": " of"
-      },
-      {
-        "id": 7515,
-        "logprob": -0.4489746,
-        "special": false,
-        "text": " numbers"
-      },
-      {
-        "id": 32,
-        "logprob": -0.6044922,
-        "special": false,
-        "text": "."
-      },
-      {
-        "id": 446,
-        "logprob": -0.63964844,
-        "special": false,
-        "text": "\n\n   "
-      },
-      {
-        "id": 499,
-        "logprob": -1.1953125,
-        "special": false,
-        "text": " :"
-      },
-      {
-        "id": 753,
-        "logprob": -0.03515625,
-        "special": false,
-        "text": "param"
-      },
-      {
-        "id": 498,
-        "logprob": -0.06311035,
-        "special": false,
-        "text": " L"
-      },
-      {
-        "id": 44,
-        "logprob": -0.003414154,
-        "special": false,
-        "text": ":"
-      },
-      {
-        "id": 1682,
-        "logprob": -1.3310547,
-        "special": false,
-        "text": " List"
+        "id": 0,
+        "logprob": null,
+        "special": true,
+        "text": "<|endoftext|>"
       }
     ],
     "top_tokens": null
   },
-  "generated_text": "\n    \"\"\"\n    Calculate the geometric mean of a list of numbers.\n\n    :param L: List"
+  "generated_text": "\n   "
 }
diff --git a/integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq_default_params.json b/integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq_default_params.json
index bf0f5146..015912f8 100644
--- a/integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq_default_params.json
+++ b/integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq_default_params.json
@@ -1,8 +1,8 @@
 {
   "details": {
     "best_of_sequences": null,
-    "finish_reason": "length",
-    "generated_tokens": 20,
+    "finish_reason": "eos_token",
+    "generated_tokens": 2,
     "prefill": [
       {
         "id": 589,
@@ -11,57 +11,57 @@
       },
       {
         "id": 3226,
-        "logprob": -8.5859375,
+        "logprob": -8.9453125,
         "text": " ge"
       },
       {
         "id": 21017,
-        "logprob": -7.5898438,
+        "logprob": -8.859375,
         "text": "ometric"
       },
       {
         "id": 81,
-        "logprob": -0.26586914,
+        "logprob": -0.21984863,
         "text": "_"
       },
       {
         "id": 6009,
-        "logprob": -1.6347656,
+        "logprob": -1.2861328,
         "text": "mean"
       },
       {
         "id": 26,
-        "logprob": -0.22705078,
+        "logprob": -0.25219727,
         "text": "("
       },
       {
         "id": 62,
-        "logprob": -5.2382812,
+        "logprob": -4.8007812,
         "text": "L"
       },
       {
         "id": 44,
-        "logprob": -3.0996094,
+        "logprob": -3.7949219,
         "text": ":"
       },
       {
         "id": 1682,
-        "logprob": -1.1025391,
+        "logprob": -0.8046875,
         "text": " List"
       },
       {
         "id": 77,
-        "logprob": -0.14294434,
+        "logprob": -0.22424316,
         "text": "["
       },
       {
         "id": 1808,
-        "logprob": -0.32226562,
+        "logprob": -0.46191406,
         "text": "float"
       },
       {
         "id": 10794,
-        "logprob": -2.8164062,
+        "logprob": -3.0253906,
         "text": "]):"
       }
     ],
@@ -74,121 +74,13 @@
         "text": "\n   "
       },
       {
-        "id": 442,
-        "logprob": -1.3134766,
-        "special": false,
-        "text": " return"
-      },
-      {
-        "id": 11665,
-        "logprob": -0.10021973,
-        "special": false,
-        "text": " reduce"
-      },
-      {
-        "id": 26,
-        "logprob": 0.0,
-        "special": false,
-        "text": "("
-      },
-      {
-        "id": 5962,
-        "logprob": 0.0,
-        "special": false,
-        "text": "lambda"
-      },
-      {
-        "id": 816,
-        "logprob": 0.0,
-        "special": false,
-        "text": " x"
-      },
-      {
-        "id": 30,
-        "logprob": 0.0,
-        "special": false,
-        "text": ","
-      },
-      {
-        "id": 533,
-        "logprob": 0.0,
-        "special": false,
-        "text": " y"
-      },
-      {
-        "id": 44,
-        "logprob": 0.0,
-        "special": false,
-        "text": ":"
-      },
-      {
-        "id": 816,
-        "logprob": 0.0,
-        "special": false,
-        "text": " x"
-      },
-      {
-        "id": 319,
-        "logprob": -0.42871094,
-        "special": false,
-        "text": " *"
-      },
-      {
-        "id": 533,
-        "logprob": 0.0,
-        "special": false,
-        "text": " y"
-      },
-      {
-        "id": 30,
-        "logprob": 0.0,
-        "special": false,
-        "text": ","
-      },
-      {
-        "id": 498,
-        "logprob": 0.0,
-        "special": false,
-        "text": " L"
-      },
-      {
-        "id": 27,
-        "logprob": 0.0,
-        "special": false,
-        "text": ")"
-      },
-      {
-        "id": 1115,
-        "logprob": 0.0,
-        "special": false,
-        "text": " **"
-      },
-      {
-        "id": 308,
-        "logprob": 0.0,
-        "special": false,
-        "text": " ("
-      },
-      {
-        "id": 35,
-        "logprob": 0.0,
-        "special": false,
-        "text": "1"
-      },
-      {
-        "id": 32,
-        "logprob": -0.31323242,
-        "special": false,
-        "text": "."
-      },
-      {
-        "id": 34,
-        "logprob": 0.0,
-        "special": false,
-        "text": "0"
+        "id": 0,
+        "logprob": null,
+        "special": true,
+        "text": "<|endoftext|>"
       }
     ],
     "top_tokens": null
   },
-  "generated_text": "\n    return reduce(lambda x, y: x * y, L) ** (1.0"
+  "generated_text": "\n   "
 }
diff --git a/integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq_load.json b/integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq_load.json
index 46a21ed8..d9072c52 100644
--- a/integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq_load.json
+++ b/integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq_load.json
@@ -2,8 +2,8 @@
   {
     "details": {
       "best_of_sequences": null,
-      "finish_reason": "length",
-      "generated_tokens": 10,
+      "finish_reason": "eos_token",
+      "generated_tokens": 2,
       "prefill": [
         {
           "id": 589,
@@ -12,57 +12,57 @@
         },
         {
           "id": 3226,
-          "logprob": -8.5859375,
+          "logprob": -8.9453125,
           "text": " ge"
         },
         {
           "id": 21017,
-          "logprob": -7.5820312,
+          "logprob": -8.8515625,
           "text": "ometric"
         },
         {
           "id": 81,
-          "logprob": -0.26708984,
+          "logprob": -0.22033691,
           "text": "_"
         },
         {
           "id": 6009,
-          "logprob": -1.6386719,
+          "logprob": -1.2939453,
           "text": "mean"
         },
         {
           "id": 26,
-          "logprob": -0.22717285,
+          "logprob": -0.25268555,
           "text": "("
         },
         {
           "id": 62,
-          "logprob": -5.234375,
+          "logprob": -4.796875,
           "text": "L"
         },
         {
           "id": 44,
-          "logprob": -3.1015625,
+          "logprob": -3.796875,
           "text": ":"
         },
         {
           "id": 1682,
-          "logprob": -1.1083984,
+          "logprob": -0.8066406,
           "text": " List"
         },
         {
           "id": 77,
-          "logprob": -0.14294434,
+          "logprob": -0.22644043,
           "text": "["
         },
         {
           "id": 1808,
-          "logprob": -0.32592773,
+          "logprob": -0.46166992,
           "text": "float"
         },
         {
           "id": 10794,
-          "logprob": -2.8164062,
+          "logprob": -3.0253906,
           "text": "]):"
         }
       ],
@@ -70,74 +70,26 @@
       "tokens": [
         {
           "id": 284,
-          "logprob": -0.12817383,
+          "logprob": -0.046844482,
           "special": false,
           "text": "\n   "
         },
         {
-          "id": 1524,
-          "logprob": -0.9863281,
-          "special": false,
-          "text": " \"\"\""
-        },
-        {
-          "id": 284,
-          "logprob": -0.7011719,
-          "special": false,
-          "text": "\n   "
-        },
-        {
-          "id": 14883,
-          "logprob": -2.2050781,
-          "special": false,
-          "text": " Calculate"
-        },
-        {
-          "id": 322,
-          "logprob": -0.2668457,
-          "special": false,
-          "text": " the"
-        },
-        {
-          "id": 3226,
-          "logprob": -0.08465576,
-          "special": false,
-          "text": " ge"
-        },
-        {
-          "id": 21017,
-          "logprob": -0.019012451,
-          "special": false,
-          "text": "ometric"
-        },
-        {
-          "id": 5651,
-          "logprob": -0.028625488,
-          "special": false,
-          "text": " mean"
-        },
-        {
-          "id": 432,
-          "logprob": -0.29418945,
-          "special": false,
-          "text": " of"
-        },
-        {
-          "id": 312,
-          "logprob": -0.3161621,
-          "special": false,
-          "text": " a"
+          "id": 0,
+          "logprob": null,
+          "special": true,
+          "text": "<|endoftext|>"
         }
       ],
       "top_tokens": null
     },
-    "generated_text": "\n    \"\"\"\n    Calculate the geometric mean of a"
+    "generated_text": "\n   "
   },
   {
     "details": {
       "best_of_sequences": null,
-      "finish_reason": "length",
-      "generated_tokens": 10,
+      "finish_reason": "eos_token",
+      "generated_tokens": 2,
       "prefill": [
         {
           "id": 589,
@@ -146,57 +98,57 @@
         },
         {
           "id": 3226,
-          "logprob": -8.5859375,
+          "logprob": -8.9375,
           "text": " ge"
         },
         {
           "id": 21017,
-          "logprob": -7.59375,
+          "logprob": -8.8515625,
           "text": "ometric"
         },
         {
           "id": 81,
-          "logprob": -0.26953125,
+          "logprob": -0.21826172,
           "text": "_"
         },
         {
           "id": 6009,
-          "logprob": -1.640625,
+          "logprob": -1.2871094,
           "text": "mean"
         },
         {
           "id": 26,
-          "logprob": -0.22705078,
+          "logprob": -0.25390625,
           "text": "("
         },
         {
           "id": 62,
-          "logprob": -5.234375,
+          "logprob": -4.8085938,
           "text": "L"
         },
         {
           "id": 44,
-          "logprob": -3.1132812,
+          "logprob": -3.7890625,
           "text": ":"
         },
         {
           "id": 1682,
-          "logprob": -1.1123047,
+          "logprob": -0.8076172,
           "text": " List"
         },
         {
           "id": 77,
-          "logprob": -0.14294434,
+          "logprob": -0.22302246,
           "text": "["
         },
         {
           "id": 1808,
-          "logprob": -0.32299805,
+          "logprob": -0.46435547,
           "text": "float"
         },
         {
           "id": 10794,
-          "logprob": -2.8164062,
+          "logprob": -3.0234375,
           "text": "]):"
         }
       ],
@@ -204,74 +156,26 @@
       "tokens": [
         {
           "id": 284,
-          "logprob": -0.12854004,
+          "logprob": -0.046722412,
           "special": false,
           "text": "\n   "
         },
         {
-          "id": 1524,
-          "logprob": -0.9897461,
-          "special": false,
-          "text": " \"\"\""
-        },
-        {
-          "id": 284,
-          "logprob": -0.69970703,
-          "special": false,
-          "text": "\n   "
-        },
-        {
-          "id": 14883,
-          "logprob": -2.2050781,
-          "special": false,
-          "text": " Calculate"
-        },
-        {
-          "id": 322,
-          "logprob": -0.2668457,
-          "special": false,
-          "text": " the"
-        },
-        {
-          "id": 3226,
-          "logprob": -0.08496094,
-          "special": false,
-          "text": " ge"
-        },
-        {
-          "id": 21017,
-          "logprob": -0.019012451,
-          "special": false,
-          "text": "ometric"
-        },
-        {
-          "id": 5651,
-          "logprob": -0.029037476,
-          "special": false,
-          "text": " mean"
-        },
-        {
-          "id": 432,
-          "logprob": -0.2939453,
-          "special": false,
-          "text": " of"
-        },
-        {
-          "id": 312,
-          "logprob": -0.31591797,
-          "special": false,
-          "text": " a"
+          "id": 0,
+          "logprob": null,
+          "special": true,
+          "text": "<|endoftext|>"
         }
       ],
       "top_tokens": null
     },
-    "generated_text": "\n    \"\"\"\n    Calculate the geometric mean of a"
+    "generated_text": "\n   "
   },
   {
     "details": {
       "best_of_sequences": null,
-      "finish_reason": "length",
-      "generated_tokens": 10,
+      "finish_reason": "eos_token",
+      "generated_tokens": 2,
       "prefill": [
         {
           "id": 589,
@@ -280,57 +184,57 @@
         },
         {
           "id": 3226,
-          "logprob": -8.5859375,
+          "logprob": -8.9453125,
           "text": " ge"
         },
         {
           "id": 21017,
-          "logprob": -7.5859375,
+          "logprob": -8.8515625,
           "text": "ometric"
         },
         {
           "id": 81,
-          "logprob": -0.26586914,
+          "logprob": -0.21813965,
           "text": "_"
         },
         {
           "id": 6009,
-          "logprob": -1.6347656,
+          "logprob": -1.2744141,
           "text": "mean"
         },
         {
           "id": 26,
-          "logprob": -0.22766113,
+          "logprob": -0.2512207,
           "text": "("
         },
         {
           "id": 62,
-          "logprob": -5.2265625,
+          "logprob": -4.8046875,
           "text": "L"
         },
         {
           "id": 44,
-          "logprob": -3.0976562,
+          "logprob": -3.7851562,
           "text": ":"
         },
         {
           "id": 1682,
-          "logprob": -1.1025391,
+          "logprob": -0.81396484,
           "text": " List"
         },
         {
           "id": 77,
-          "logprob": -0.1427002,
+          "logprob": -0.22570801,
           "text": "["
         },
         {
           "id": 1808,
-          "logprob": -0.32592773,
+          "logprob": -0.46044922,
           "text": "float"
         },
         {
           "id": 10794,
-          "logprob": -2.8164062,
+          "logprob": -3.0234375,
           "text": "]):"
         }
       ],
@@ -338,74 +242,26 @@
       "tokens": [
         {
           "id": 284,
-          "logprob": -0.13012695,
+          "logprob": -0.04650879,
           "special": false,
           "text": "\n   "
         },
         {
-          "id": 1524,
-          "logprob": -0.98046875,
-          "special": false,
-          "text": " \"\"\""
-        },
-        {
-          "id": 284,
-          "logprob": -0.69921875,
-          "special": false,
-          "text": "\n   "
-        },
-        {
-          "id": 14883,
-          "logprob": -2.1992188,
-          "special": false,
-          "text": " Calculate"
-        },
-        {
-          "id": 322,
-          "logprob": -0.2668457,
-          "special": false,
-          "text": " the"
-        },
-        {
-          "id": 3226,
-          "logprob": -0.083496094,
-          "special": false,
-          "text": " ge"
-        },
-        {
-          "id": 21017,
-          "logprob": -0.01902771,
-          "special": false,
-          "text": "ometric"
-        },
-        {
-          "id": 5651,
-          "logprob": -0.029006958,
-          "special": false,
-          "text": " mean"
-        },
-        {
-          "id": 432,
-          "logprob": -0.29248047,
-          "special": false,
-          "text": " of"
-        },
-        {
-          "id": 312,
-          "logprob": -0.3161621,
-          "special": false,
-          "text": " a"
+          "id": 0,
+          "logprob": null,
+          "special": true,
+          "text": "<|endoftext|>"
         }
       ],
       "top_tokens": null
     },
-    "generated_text": "\n    \"\"\"\n    Calculate the geometric mean of a"
+    "generated_text": "\n   "
   },
   {
     "details": {
       "best_of_sequences": null,
-      "finish_reason": "length",
-      "generated_tokens": 10,
+      "finish_reason": "eos_token",
+      "generated_tokens": 2,
       "prefill": [
         {
           "id": 589,
@@ -414,57 +270,57 @@
         },
         {
           "id": 3226,
-          "logprob": -8.5859375,
+          "logprob": -8.9453125,
           "text": " ge"
         },
         {
           "id": 21017,
-          "logprob": -7.5859375,
+          "logprob": -8.8515625,
           "text": "ometric"
         },
         {
           "id": 81,
-          "logprob": -0.26904297,
+          "logprob": -0.21960449,
           "text": "_"
         },
         {
           "id": 6009,
-          "logprob": -1.6386719,
+          "logprob": -1.2890625,
           "text": "mean"
         },
         {
           "id": 26,
-          "logprob": -0.22705078,
+          "logprob": -0.25073242,
           "text": "("
         },
         {
           "id": 62,
-          "logprob": -5.234375,
+          "logprob": -4.8085938,
           "text": "L"
         },
         {
           "id": 44,
-          "logprob": -3.1132812,
+          "logprob": -3.8046875,
           "text": ":"
         },
         {
           "id": 1682,
-          "logprob": -1.1074219,
+          "logprob": -0.8071289,
           "text": " List"
         },
         {
           "id": 77,
-          "logprob": -0.14477539,
+          "logprob": -0.22570801,
           "text": "["
         },
         {
           "id": 1808,
-          "logprob": -0.3256836,
+          "logprob": -0.46118164,
           "text": "float"
         },
         {
           "id": 10794,
-          "logprob": -2.8027344,
+          "logprob": -3.0097656,
           "text": "]):"
         }
       ],
@@ -472,67 +328,19 @@
       "tokens": [
         {
           "id": 284,
-          "logprob": -0.12915039,
+          "logprob": -0.046539307,
           "special": false,
           "text": "\n   "
         },
         {
-          "id": 1524,
-          "logprob": -0.98535156,
-          "special": false,
-          "text": " \"\"\""
-        },
-        {
-          "id": 284,
-          "logprob": -0.69921875,
-          "special": false,
-          "text": "\n   "
-        },
-        {
-          "id": 14883,
-          "logprob": -2.2011719,
-          "special": false,
-          "text": " Calculate"
-        },
-        {
-          "id": 322,
-          "logprob": -0.26708984,
-          "special": false,
-          "text": " the"
-        },
-        {
-          "id": 3226,
-          "logprob": -0.08502197,
-          "special": false,
-          "text": " ge"
-        },
-        {
-          "id": 21017,
-          "logprob": -0.019012451,
-          "special": false,
-          "text": "ometric"
-        },
-        {
-          "id": 5651,
-          "logprob": -0.028625488,
-          "special": false,
-          "text": " mean"
-        },
-        {
-          "id": 432,
-          "logprob": -0.29589844,
-          "special": false,
-          "text": " of"
-        },
-        {
-          "id": 312,
-          "logprob": -0.31591797,
-          "special": false,
-          "text": " a"
+          "id": 0,
+          "logprob": null,
+          "special": true,
+          "text": "<|endoftext|>"
         }
       ],
       "top_tokens": null
     },
-    "generated_text": "\n    \"\"\"\n    Calculate the geometric mean of a"
+    "generated_text": "\n   "
   }
 ]
diff --git a/integration-tests/models/__snapshots__/test_grammar_response_format_llama/test_grammar_response_format_llama_json.json b/integration-tests/models/__snapshots__/test_grammar_response_format_llama/test_grammar_response_format_llama_json.json
new file mode 100644
index 00000000..83390832
--- /dev/null
+++ b/integration-tests/models/__snapshots__/test_grammar_response_format_llama/test_grammar_response_format_llama_json.json
@@ -0,0 +1,23 @@
+{
+  "choices": [
+    {
+      "finish_reason": "eos_token",
+      "index": 0,
+      "logprobs": null,
+      "message": {
+        "content": "{\n  \"temperature\": [\n    35,\n    34,\n    36\n  ],\n  \"unit\": \"°c\"\n}",
+        "role": "assistant"
+      }
+    }
+  ],
+  "created": 1718044128,
+  "id": "",
+  "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
+  "object": "text_completion",
+  "system_fingerprint": "2.0.5-dev0-native",
+  "usage": {
+    "completion_tokens": 39,
+    "prompt_tokens": 136,
+    "total_tokens": 175
+  }
+}
diff --git a/integration-tests/models/__snapshots__/test_idefics/test_idefics_two_images.json b/integration-tests/models/__snapshots__/test_idefics/test_idefics_two_images.json
new file mode 100644
index 00000000..a4727707
--- /dev/null
+++ b/integration-tests/models/__snapshots__/test_idefics/test_idefics_two_images.json
@@ -0,0 +1,85 @@
+{
+  "details": {
+    "best_of_sequences": null,
+    "finish_reason": "eos_token",
+    "generated_tokens": 12,
+    "prefill": [],
+    "seed": null,
+    "tokens": [
+      {
+        "id": 450,
+        "logprob": -0.26342773,
+        "special": false,
+        "text": " The"
+      },
+      {
+        "id": 21282,
+        "logprob": -0.01838684,
+        "special": false,
+        "text": " cow"
+      },
+      {
+        "id": 322,
+        "logprob": -0.18041992,
+        "special": false,
+        "text": " and"
+      },
+      {
+        "id": 521,
+        "logprob": -0.62841797,
+        "special": false,
+        "text": " ch"
+      },
+      {
+        "id": 21475,
+        "logprob": -0.0037956238,
+        "special": false,
+        "text": "icken"
+      },
+      {
+        "id": 526,
+        "logprob": -0.018737793,
+        "special": false,
+        "text": " are"
+      },
+      {
+        "id": 373,
+        "logprob": -1.0820312,
+        "special": false,
+        "text": " on"
+      },
+      {
+        "id": 263,
+        "logprob": -0.5083008,
+        "special": false,
+        "text": " a"
+      },
+      {
+        "id": 25695,
+        "logprob": -0.07128906,
+        "special": false,
+        "text": " beach"
+      },
+      {
+        "id": 29889,
+        "logprob": -0.12573242,
+        "special": false,
+        "text": "."
+      },
+      {
+        "id": 32002,
+        "logprob": -0.0029792786,
+        "special": true,
+        "text": "<end_of_utterance>"
+      },
+      {
+        "id": 2,
+        "logprob": -0.00024962425,
+        "special": true,
+        "text": "</s>"
+      }
+    ],
+    "top_tokens": null
+  },
+  "generated_text": " The cow and chicken are on a beach."
+}
diff --git a/integration-tests/models/__snapshots__/test_idefics2/test_flash_idefics2_next_all_params.json b/integration-tests/models/__snapshots__/test_idefics2/test_flash_idefics2_next_all_params.json
index 45601505..dab437b9 100644
--- a/integration-tests/models/__snapshots__/test_idefics2/test_flash_idefics2_next_all_params.json
+++ b/integration-tests/models/__snapshots__/test_idefics2/test_flash_idefics2_next_all_params.json
@@ -30,7 +30,7 @@
       },
       {
         "id": 264,
-        "logprob": -0.37573242,
+        "logprob": -0.38061523,
         "special": false,
         "text": " a"
       },
@@ -42,7 +42,7 @@
       },
       {
         "id": 4480,
-        "logprob": -0.3322754,
+        "logprob": -0.26782227,
         "special": false,
         "text": " feature"
       },
@@ -78,7 +78,7 @@
       },
       {
         "id": 13,
-        "logprob": 0.0,
+        "logprob": -0.10632324,
         "special": false,
         "text": "\n"
       }
diff --git a/integration-tests/models/__snapshots__/test_idefics2/test_flash_idefics2_next_load.json b/integration-tests/models/__snapshots__/test_idefics2/test_flash_idefics2_next_load.json
index 4bc90896..7f1875e0 100644
--- a/integration-tests/models/__snapshots__/test_idefics2/test_flash_idefics2_next_load.json
+++ b/integration-tests/models/__snapshots__/test_idefics2/test_flash_idefics2_next_load.json
@@ -37,7 +37,7 @@
         },
         {
           "id": 32001,
-          "logprob": -19.484375,
+          "logprob": -19.46875,
           "text": "<image>"
         },
         {
@@ -57,7 +57,7 @@
         },
         {
           "id": 32001,
-          "logprob": -20.234375,
+          "logprob": -20.21875,
           "text": "<image>"
         },
         {
@@ -65,11 +65,1785 @@
           "logprob": -16.421875,
           "text": "<image>"
         },
+        {
+          "id": 32001,
+          "logprob": -19.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -23.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -15.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -22.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -23.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -23.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -14.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -23.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -23.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -23.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -15.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -3.015625,
+          "text": "<fake_token_around_image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -22.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -15.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -15.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -15.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -14.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -15.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -14.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -3.2988281,
+          "text": "<fake_token_around_image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -25.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -15.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -15.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -15.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -15.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -15.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -2.7207031,
+          "text": "<fake_token_around_image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -23.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -22.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.25,
+          "text": "<image>"
+        },
         {
           "id": 32001,
           "logprob": -19.828125,
           "text": "<image>"
         },
+        {
+          "id": 32001,
+          "logprob": -15.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -15.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -15.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -3.0917969,
+          "text": "<fake_token_around_image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -25.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -15.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -15.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -22.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -22.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -22.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -15.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -14.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -13.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -15.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -15.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -15.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -2.0332031,
+          "text": "<fake_token_around_image>"
+        },
+        {
+          "id": 12018,
+          "logprob": -12.078125,
+          "text": "Write"
+        },
+        {
+          "id": 528,
+          "logprob": -10.09375,
+          "text": "me"
+        },
+        {
+          "id": 264,
+          "logprob": -0.103393555,
+          "text": "a"
+        },
+        {
+          "id": 2485,
+          "logprob": -4.5742188,
+          "text": "short"
+        },
+        {
+          "id": 2838,
+          "logprob": -0.23815918,
+          "text": "story"
+        },
+        {
+          "id": 32002,
+          "logprob": -10.9765625,
+          "text": "<end_of_utterance>"
+        },
+        {
+          "id": 259,
+          "logprob": -20.34375,
+          "text": " "
+        },
+        {
+          "id": 13,
+          "logprob": -8.53125,
+          "text": "\n"
+        },
+        {
+          "id": 7226,
+          "logprob": -10.4765625,
+          "text": "Ass"
+        },
+        {
+          "id": 11143,
+          "logprob": -13.6015625,
+          "text": "istant"
+        },
+        {
+          "id": 28747,
+          "logprob": -0.008514404,
+          "text": ":"
+        }
+      ],
+      "seed": null,
+      "tokens": [
+        {
+          "id": 330,
+          "logprob": -0.09289551,
+          "special": false,
+          "text": " A"
+        },
+        {
+          "id": 13088,
+          "logprob": -0.6743164,
+          "special": false,
+          "text": " chicken"
+        },
+        {
+          "id": 349,
+          "logprob": -0.31396484,
+          "special": false,
+          "text": " is"
+        },
+        {
+          "id": 6398,
+          "logprob": -0.051727295,
+          "special": false,
+          "text": " sitting"
+        },
+        {
+          "id": 356,
+          "logprob": -0.34448242,
+          "special": false,
+          "text": " on"
+        },
+        {
+          "id": 264,
+          "logprob": -0.1194458,
+          "special": false,
+          "text": " a"
+        },
+        {
+          "id": 17972,
+          "logprob": -0.03237915,
+          "special": false,
+          "text": " pile"
+        },
+        {
+          "id": 302,
+          "logprob": -0.00018751621,
+          "special": false,
+          "text": " of"
+        },
+        {
+          "id": 2445,
+          "logprob": -0.07043457,
+          "special": false,
+          "text": " money"
+        },
+        {
+          "id": 28723,
+          "logprob": -0.00422287,
+          "special": false,
+          "text": "."
+        }
+      ],
+      "top_tokens": null
+    },
+    "generated_text": " A chicken is sitting on a pile of money."
+  },
+  {
+    "details": {
+      "best_of_sequences": null,
+      "finish_reason": "length",
+      "generated_tokens": 10,
+      "prefill": [
+        {
+          "id": 1,
+          "logprob": null,
+          "text": "<s>"
+        },
+        {
+          "id": 1247,
+          "logprob": -5.2382812,
+          "text": "User"
+        },
+        {
+          "id": 28747,
+          "logprob": -6.9492188,
+          "text": ":"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.234375,
+          "text": "<fake_token_around_image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.84375,
+          "text": "<image>"
+        },
         {
           "id": 32001,
           "logprob": -23.25,
@@ -102,7 +1876,7 @@
         },
         {
           "id": 32001,
-          "logprob": -21.015625,
+          "logprob": -21.03125,
           "text": "<image>"
         },
         {
@@ -112,12 +1886,12 @@
         },
         {
           "id": 32001,
-          "logprob": -16.015625,
+          "logprob": -16.03125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.0625,
+          "logprob": -19.046875,
           "text": "<image>"
         },
         {
@@ -127,12 +1901,7 @@
         },
         {
           "id": 32001,
-          "logprob": -23.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.40625,
+          "logprob": -23.609375,
           "text": "<image>"
         },
         {
@@ -142,7 +1911,12 @@
         },
         {
           "id": 32001,
-          "logprob": -20.84375,
+          "logprob": -20.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.875,
           "text": "<image>"
         },
         {
@@ -152,12 +1926,12 @@
         },
         {
           "id": 32001,
-          "logprob": -19.984375,
+          "logprob": -19.96875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.21875,
+          "logprob": -18.234375,
           "text": "<image>"
         },
         {
@@ -167,7 +1941,7 @@
         },
         {
           "id": 32001,
-          "logprob": -21.203125,
+          "logprob": -21.1875,
           "text": "<image>"
         },
         {
@@ -182,7 +1956,7 @@
         },
         {
           "id": 32001,
-          "logprob": -18.984375,
+          "logprob": -19.03125,
           "text": "<image>"
         },
         {
@@ -197,17 +1971,17 @@
         },
         {
           "id": 32001,
-          "logprob": -18.0,
+          "logprob": -17.96875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.828125,
+          "logprob": -18.859375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.9375,
+          "logprob": -17.921875,
           "text": "<image>"
         },
         {
@@ -217,12 +1991,12 @@
         },
         {
           "id": 32001,
-          "logprob": -18.640625,
+          "logprob": -18.65625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.125,
+          "logprob": -20.140625,
           "text": "<image>"
         },
         {
@@ -247,7 +2021,7 @@
         },
         {
           "id": 32001,
-          "logprob": -17.4375,
+          "logprob": -17.421875,
           "text": "<image>"
         },
         {
@@ -257,7 +2031,7 @@
         },
         {
           "id": 32001,
-          "logprob": -23.015625,
+          "logprob": -23.0,
           "text": "<image>"
         },
         {
@@ -282,7 +2056,7 @@
         },
         {
           "id": 32001,
-          "logprob": -18.40625,
+          "logprob": -18.421875,
           "text": "<image>"
         },
         {
@@ -292,17 +2066,17 @@
         },
         {
           "id": 32001,
-          "logprob": -18.34375,
+          "logprob": -18.328125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.140625,
+          "logprob": -17.125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.671875,
+          "logprob": -18.65625,
           "text": "<image>"
         },
         {
@@ -317,7 +2091,7 @@
         },
         {
           "id": 32001,
-          "logprob": -18.1875,
+          "logprob": -18.15625,
           "text": "<image>"
         },
         {
@@ -337,12 +2111,7 @@
         },
         {
           "id": 32001,
-          "logprob": -18.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.9375,
+          "logprob": -18.703125,
           "text": "<image>"
         },
         {
@@ -351,883 +2120,38 @@
           "text": "<image>"
         },
         {
-          "id": 32001,
-          "logprob": -19.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.1875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -15.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -15.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -15.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -14.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -15.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -14.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -15.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -15.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -22.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.1875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.1875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -15.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.375,
-          "text": "<image>"
+          "id": 32000,
+          "logprob": -3.015625,
+          "text": "<fake_token_around_image>"
         },
         {
           "id": 32001,
-          "logprob": -20.140625,
+          "logprob": -22.109375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.140625,
+          "logprob": -18.96875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.6875,
+          "logprob": -20.125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.453125,
+          "logprob": -17.125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.171875,
+          "logprob": -17.8125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.78125,
+          "logprob": -19.3125,
           "text": "<image>"
         },
         {
@@ -1237,157 +2161,27 @@
         },
         {
           "id": 32001,
-          "logprob": -17.078125,
+          "logprob": -16.3125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.109375,
+          "logprob": -19.375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.171875,
+          "logprob": -20.046875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.453125,
+          "logprob": -20.828125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -15.8828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -15.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.75,
+          "logprob": -15.8046875,
           "text": "<image>"
         },
         {
@@ -1397,7 +2191,12 @@
         },
         {
           "id": 32001,
-          "logprob": -19.46875,
+          "logprob": -19.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.875,
           "text": "<image>"
         },
         {
@@ -1407,22 +2206,297 @@
         },
         {
           "id": 32001,
-          "logprob": -22.421875,
+          "logprob": -20.515625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.9375,
+          "logprob": -19.171875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.671875,
+          "logprob": -19.296875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.890625,
+          "logprob": -16.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -15.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -15.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -14.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -15.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -14.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -3.2988281,
+          "text": "<fake_token_around_image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -25.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.203125,
           "text": "<image>"
         },
         {
@@ -1432,12 +2506,732 @@
         },
         {
           "id": 32001,
-          "logprob": -17.5,
+          "logprob": -15.75,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.90625,
+          "logprob": -16.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -15.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -15.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -15.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -15.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -2.7207031,
+          "text": "<fake_token_around_image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -23.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -22.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -15.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -15.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -15.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -3.0917969,
+          "text": "<fake_token_around_image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -25.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -15.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -15.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -22.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.921875,
           "text": "<image>"
         },
         {
@@ -1452,37 +3246,37 @@
         },
         {
           "id": 32001,
-          "logprob": -16.40625,
+          "logprob": -16.375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.453125,
+          "logprob": -18.4375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.234375,
+          "logprob": -20.265625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -22.28125,
+          "logprob": -22.296875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.515625,
+          "logprob": -18.484375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.4296875,
+          "logprob": -15.390625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.765625,
+          "logprob": -19.75,
           "text": "<image>"
         },
         {
@@ -1492,32 +3286,32 @@
         },
         {
           "id": 32001,
-          "logprob": -21.46875,
+          "logprob": -21.609375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.875,
+          "logprob": -18.828125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.859375,
+          "logprob": -20.828125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.078125,
+          "logprob": -17.015625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.4375,
+          "logprob": -16.40625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.015625,
+          "logprob": -21.046875,
           "text": "<image>"
         },
         {
@@ -1532,22 +3326,22 @@
         },
         {
           "id": 32001,
-          "logprob": -21.484375,
+          "logprob": -21.515625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.015625,
+          "logprob": -20.0,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.84375,
+          "logprob": -18.78125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.40625,
+          "logprob": -16.375,
           "text": "<image>"
         },
         {
@@ -1557,57 +3351,57 @@
         },
         {
           "id": 32001,
-          "logprob": -16.65625,
+          "logprob": -16.703125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -13.6328125,
+          "logprob": -13.625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.4140625,
+          "logprob": -15.375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.546875,
+          "logprob": -17.515625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.859375,
+          "logprob": -21.921875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.65625,
+          "logprob": -15.640625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.484375,
+          "logprob": -16.46875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.359375,
+          "logprob": -16.421875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.9375,
+          "logprob": -19.890625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.875,
+          "logprob": -17.890625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.453125,
+          "logprob": -17.40625,
           "text": "<image>"
         },
         {
@@ -1617,72 +3411,72 @@
         },
         {
           "id": 32001,
-          "logprob": -19.171875,
+          "logprob": -19.1875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.9921875,
+          "logprob": -15.9609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -2.0429688,
+          "logprob": -2.0332031,
           "text": "<fake_token_around_image>"
         },
         {
           "id": 12018,
-          "logprob": -12.03125,
+          "logprob": -12.078125,
           "text": "Write"
         },
         {
           "id": 528,
-          "logprob": -10.25,
+          "logprob": -10.109375,
           "text": "me"
         },
         {
           "id": 264,
-          "logprob": -0.10437012,
+          "logprob": -0.103515625,
           "text": "a"
         },
         {
           "id": 2485,
-          "logprob": -4.5742188,
+          "logprob": -4.5664062,
           "text": "short"
         },
         {
           "id": 2838,
-          "logprob": -0.2277832,
+          "logprob": -0.23864746,
           "text": "story"
         },
         {
           "id": 32002,
-          "logprob": -10.84375,
+          "logprob": -10.9609375,
           "text": "<end_of_utterance>"
         },
         {
           "id": 259,
-          "logprob": -20.1875,
+          "logprob": -20.34375,
           "text": " "
         },
         {
           "id": 13,
-          "logprob": -8.7578125,
+          "logprob": -8.5546875,
           "text": "\n"
         },
         {
           "id": 7226,
-          "logprob": -10.421875,
+          "logprob": -10.484375,
           "text": "Ass"
         },
         {
           "id": 11143,
-          "logprob": -13.640625,
+          "logprob": -13.6015625,
           "text": "istant"
         },
         {
           "id": 28747,
-          "logprob": -0.005619049,
+          "logprob": -0.008308411,
           "text": ":"
         }
       ],
@@ -1690,61 +3484,61 @@
       "tokens": [
         {
           "id": 330,
-          "logprob": -0.12939453,
+          "logprob": -0.09448242,
           "special": false,
           "text": " A"
         },
         {
           "id": 13088,
-          "logprob": -0.6660156,
+          "logprob": -0.6743164,
           "special": false,
           "text": " chicken"
         },
         {
           "id": 349,
-          "logprob": -0.29638672,
+          "logprob": -0.31201172,
           "special": false,
           "text": " is"
         },
         {
           "id": 6398,
-          "logprob": -0.05960083,
+          "logprob": -0.051635742,
           "special": false,
           "text": " sitting"
         },
         {
           "id": 356,
-          "logprob": -0.26953125,
+          "logprob": -0.34033203,
           "special": false,
           "text": " on"
         },
         {
           "id": 264,
-          "logprob": -0.1427002,
+          "logprob": -0.1194458,
           "special": false,
           "text": " a"
         },
         {
           "id": 17972,
-          "logprob": -0.040649414,
+          "logprob": -0.032562256,
           "special": false,
           "text": " pile"
         },
         {
           "id": 302,
-          "logprob": -0.0002708435,
+          "logprob": -0.00018763542,
           "special": false,
           "text": " of"
         },
         {
           "id": 2445,
-          "logprob": -0.09429932,
+          "logprob": -0.07122803,
           "special": false,
           "text": " money"
         },
         {
           "id": 28723,
-          "logprob": -0.006931305,
+          "logprob": -0.0041007996,
           "special": false,
           "text": "."
         }
@@ -1766,12 +3560,12 @@
         },
         {
           "id": 1247,
-          "logprob": -5.234375,
+          "logprob": -5.2382812,
           "text": "User"
         },
         {
           "id": 28747,
-          "logprob": -6.9648438,
+          "logprob": -6.9492188,
           "text": ":"
         },
         {
@@ -1781,12 +3575,12 @@
         },
         {
           "id": 32001,
-          "logprob": -18.96875,
+          "logprob": -18.984375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.1875,
+          "logprob": -18.171875,
           "text": "<image>"
         },
         {
@@ -1811,7 +3605,7 @@
         },
         {
           "id": 32001,
-          "logprob": -20.234375,
+          "logprob": -20.21875,
           "text": "<image>"
         },
         {
@@ -1821,7 +3615,7 @@
         },
         {
           "id": 32001,
-          "logprob": -19.828125,
+          "logprob": -19.84375,
           "text": "<image>"
         },
         {
@@ -1856,22 +3650,22 @@
         },
         {
           "id": 32001,
-          "logprob": -21.015625,
+          "logprob": -21.03125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.4375,
+          "logprob": -20.421875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.015625,
+          "logprob": -16.03125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.0625,
+          "logprob": -19.046875,
           "text": "<image>"
         },
         {
@@ -1886,7 +3680,7 @@
         },
         {
           "id": 32001,
-          "logprob": -20.40625,
+          "logprob": -20.421875,
           "text": "<image>"
         },
         {
@@ -1896,7 +3690,7 @@
         },
         {
           "id": 32001,
-          "logprob": -20.84375,
+          "logprob": -20.875,
           "text": "<image>"
         },
         {
@@ -1906,12 +3700,12 @@
         },
         {
           "id": 32001,
-          "logprob": -19.984375,
+          "logprob": -19.96875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.21875,
+          "logprob": -18.234375,
           "text": "<image>"
         },
         {
@@ -1921,7 +3715,7 @@
         },
         {
           "id": 32001,
-          "logprob": -21.203125,
+          "logprob": -21.1875,
           "text": "<image>"
         },
         {
@@ -1936,7 +3730,7 @@
         },
         {
           "id": 32001,
-          "logprob": -18.984375,
+          "logprob": -19.03125,
           "text": "<image>"
         },
         {
@@ -1951,17 +3745,17 @@
         },
         {
           "id": 32001,
-          "logprob": -18.0,
+          "logprob": -17.96875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.828125,
+          "logprob": -18.859375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.9375,
+          "logprob": -17.921875,
           "text": "<image>"
         },
         {
@@ -1971,12 +3765,12 @@
         },
         {
           "id": 32001,
-          "logprob": -18.640625,
+          "logprob": -18.65625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.125,
+          "logprob": -20.140625,
           "text": "<image>"
         },
         {
@@ -1996,12 +3790,12 @@
         },
         {
           "id": 32001,
-          "logprob": -23.203125,
+          "logprob": -23.21875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.4375,
+          "logprob": -17.421875,
           "text": "<image>"
         },
         {
@@ -2011,7 +3805,7 @@
         },
         {
           "id": 32001,
-          "logprob": -23.015625,
+          "logprob": -23.0,
           "text": "<image>"
         },
         {
@@ -2036,7 +3830,7 @@
         },
         {
           "id": 32001,
-          "logprob": -18.40625,
+          "logprob": -18.421875,
           "text": "<image>"
         },
         {
@@ -2046,7 +3840,7 @@
         },
         {
           "id": 32001,
-          "logprob": -18.34375,
+          "logprob": -18.328125,
           "text": "<image>"
         },
         {
@@ -2056,7 +3850,7 @@
         },
         {
           "id": 32001,
-          "logprob": -18.671875,
+          "logprob": -18.65625,
           "text": "<image>"
         },
         {
@@ -2071,7 +3865,7 @@
         },
         {
           "id": 32001,
-          "logprob": -18.1875,
+          "logprob": -18.15625,
           "text": "<image>"
         },
         {
@@ -2081,7 +3875,7 @@
         },
         {
           "id": 32001,
-          "logprob": -20.140625,
+          "logprob": -20.15625,
           "text": "<image>"
         },
         {
@@ -2089,1643 +3883,29 @@
           "logprob": -18.96875,
           "text": "<image>"
         },
-        {
-          "id": 32001,
-          "logprob": -18.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.1875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -15.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -15.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -15.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -14.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -15.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -14.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.203125,
-          "text": "<image>"
-        },
         {
           "id": 32001,
           "logprob": -18.703125,
           "text": "<image>"
         },
-        {
-          "id": 32001,
-          "logprob": -19.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -15.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -15.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -22.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.1875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -15.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -15.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -15.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -22.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -22.1875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -22.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -15.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -14.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -13.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -15.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -15.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -15.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -2.0429688,
-          "text": "<fake_token_around_image>"
-        },
-        {
-          "id": 12018,
-          "logprob": -12.03125,
-          "text": "Write"
-        },
-        {
-          "id": 528,
-          "logprob": -10.2578125,
-          "text": "me"
-        },
-        {
-          "id": 264,
-          "logprob": -0.10418701,
-          "text": "a"
-        },
-        {
-          "id": 2485,
-          "logprob": -4.5664062,
-          "text": "short"
-        },
-        {
-          "id": 2838,
-          "logprob": -0.22741699,
-          "text": "story"
-        },
-        {
-          "id": 32002,
-          "logprob": -10.8515625,
-          "text": "<end_of_utterance>"
-        },
-        {
-          "id": 259,
-          "logprob": -20.203125,
-          "text": " "
-        },
-        {
-          "id": 13,
-          "logprob": -8.7421875,
-          "text": "\n"
-        },
-        {
-          "id": 7226,
-          "logprob": -10.4140625,
-          "text": "Ass"
-        },
-        {
-          "id": 11143,
-          "logprob": -13.6328125,
-          "text": "istant"
-        },
-        {
-          "id": 28747,
-          "logprob": -0.005580902,
-          "text": ":"
-        }
-      ],
-      "seed": null,
-      "tokens": [
-        {
-          "id": 330,
-          "logprob": -0.1295166,
-          "special": false,
-          "text": " A"
-        },
-        {
-          "id": 13088,
-          "logprob": -0.6669922,
-          "special": false,
-          "text": " chicken"
-        },
-        {
-          "id": 349,
-          "logprob": -0.29711914,
-          "special": false,
-          "text": " is"
-        },
-        {
-          "id": 6398,
-          "logprob": -0.059936523,
-          "special": false,
-          "text": " sitting"
-        },
-        {
-          "id": 356,
-          "logprob": -0.27124023,
-          "special": false,
-          "text": " on"
-        },
-        {
-          "id": 264,
-          "logprob": -0.140625,
-          "special": false,
-          "text": " a"
-        },
-        {
-          "id": 17972,
-          "logprob": -0.04058838,
-          "special": false,
-          "text": " pile"
-        },
-        {
-          "id": 302,
-          "logprob": -0.00027012825,
-          "special": false,
-          "text": " of"
-        },
-        {
-          "id": 2445,
-          "logprob": -0.09503174,
-          "special": false,
-          "text": " money"
-        },
-        {
-          "id": 28723,
-          "logprob": -0.006942749,
-          "special": false,
-          "text": "."
-        }
-      ],
-      "top_tokens": null
-    },
-    "generated_text": " A chicken is sitting on a pile of money."
-  },
-  {
-    "details": {
-      "best_of_sequences": null,
-      "finish_reason": "length",
-      "generated_tokens": 10,
-      "prefill": [
-        {
-          "id": 1,
-          "logprob": null,
-          "text": "<s>"
-        },
-        {
-          "id": 1247,
-          "logprob": -5.2460938,
-          "text": "User"
-        },
-        {
-          "id": 28747,
-          "logprob": -6.9570312,
-          "text": ":"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.234375,
-          "text": "<fake_token_around_image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.1875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -23.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -15.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -22.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -23.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -23.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.84375,
-          "text": "<image>"
-        },
         {
           "id": 32001,
           "logprob": -17.921875,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -3.015625,
+          "text": "<fake_token_around_image>"
+        },
         {
           "id": 32001,
-          "logprob": -19.1875,
+          "logprob": -22.109375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.640625,
+          "logprob": -18.96875,
           "text": "<image>"
         },
         {
@@ -3733,76 +3913,6 @@
           "logprob": -20.125,
           "text": "<image>"
         },
-        {
-          "id": 32001,
-          "logprob": -19.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -14.8828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -23.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -23.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -23.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.34375,
-          "text": "<image>"
-        },
         {
           "id": 32001,
           "logprob": -17.125,
@@ -3810,932 +3920,12 @@
         },
         {
           "id": 32001,
-          "logprob": -18.671875,
+          "logprob": -17.8125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.1875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.1875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -15.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.1875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -15.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -15.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -14.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -15.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -14.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -15.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -15.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.1875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -22.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.1875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.1875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -15.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.765625,
+          "logprob": -19.3125,
           "text": "<image>"
         },
         {
@@ -4743,1385 +3933,11 @@
           "logprob": -19.65625,
           "text": "<image>"
         },
-        {
-          "id": 32001,
-          "logprob": -17.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -15.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -15.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -22.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -22.1875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -22.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -15.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -14.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -13.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -15.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -15.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -15.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -2.0429688,
-          "text": "<fake_token_around_image>"
-        },
-        {
-          "id": 12018,
-          "logprob": -12.0390625,
-          "text": "Write"
-        },
-        {
-          "id": 528,
-          "logprob": -10.25,
-          "text": "me"
-        },
-        {
-          "id": 264,
-          "logprob": -0.10443115,
-          "text": "a"
-        },
-        {
-          "id": 2485,
-          "logprob": -4.5742188,
-          "text": "short"
-        },
-        {
-          "id": 2838,
-          "logprob": -0.22729492,
-          "text": "story"
-        },
-        {
-          "id": 32002,
-          "logprob": -10.84375,
-          "text": "<end_of_utterance>"
-        },
-        {
-          "id": 259,
-          "logprob": -20.1875,
-          "text": " "
-        },
-        {
-          "id": 13,
-          "logprob": -8.7578125,
-          "text": "\n"
-        },
-        {
-          "id": 7226,
-          "logprob": -10.4140625,
-          "text": "Ass"
-        },
-        {
-          "id": 11143,
-          "logprob": -13.6328125,
-          "text": "istant"
-        },
-        {
-          "id": 28747,
-          "logprob": -0.0056533813,
-          "text": ":"
-        }
-      ],
-      "seed": null,
-      "tokens": [
-        {
-          "id": 330,
-          "logprob": -0.12963867,
-          "special": false,
-          "text": " A"
-        },
-        {
-          "id": 13088,
-          "logprob": -0.6660156,
-          "special": false,
-          "text": " chicken"
-        },
-        {
-          "id": 349,
-          "logprob": -0.29516602,
-          "special": false,
-          "text": " is"
-        },
-        {
-          "id": 6398,
-          "logprob": -0.060028076,
-          "special": false,
-          "text": " sitting"
-        },
-        {
-          "id": 356,
-          "logprob": -0.27075195,
-          "special": false,
-          "text": " on"
-        },
-        {
-          "id": 264,
-          "logprob": -0.1427002,
-          "special": false,
-          "text": " a"
-        },
-        {
-          "id": 17972,
-          "logprob": -0.04067993,
-          "special": false,
-          "text": " pile"
-        },
-        {
-          "id": 302,
-          "logprob": -0.000269413,
-          "special": false,
-          "text": " of"
-        },
-        {
-          "id": 2445,
-          "logprob": -0.09387207,
-          "special": false,
-          "text": " money"
-        },
-        {
-          "id": 28723,
-          "logprob": -0.0069236755,
-          "special": false,
-          "text": "."
-        }
-      ],
-      "top_tokens": null
-    },
-    "generated_text": " A chicken is sitting on a pile of money."
-  },
-  {
-    "details": {
-      "best_of_sequences": null,
-      "finish_reason": "length",
-      "generated_tokens": 10,
-      "prefill": [
-        {
-          "id": 1,
-          "logprob": null,
-          "text": "<s>"
-        },
-        {
-          "id": 1247,
-          "logprob": -5.2421875,
-          "text": "User"
-        },
-        {
-          "id": 28747,
-          "logprob": -6.9570312,
-          "text": ":"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.234375,
-          "text": "<fake_token_around_image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.1875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -23.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -15.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -22.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -23.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -23.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.1875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -14.8828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -23.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -23.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -23.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.1875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.921875,
-          "text": "<image>"
-        },
         {
           "id": 32001,
           "logprob": -16.3125,
           "text": "<image>"
         },
-        {
-          "id": 32001,
-          "logprob": -19.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.1875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -15.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -15.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -15.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -14.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -15.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -14.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -15.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -15.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.515625,
-          "text": "<image>"
-        },
         {
           "id": 32001,
           "logprob": -19.375,
@@ -6129,152 +3945,27 @@
         },
         {
           "id": 32001,
-          "logprob": -17.796875,
+          "logprob": -20.046875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.03125,
+          "logprob": -20.828125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.671875,
+          "logprob": -15.8046875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.15625,
+          "logprob": -16.25,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -22.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.546875,
+          "logprob": -19.953125,
           "text": "<image>"
         },
         {
@@ -6284,37 +3975,42 @@
         },
         {
           "id": 32001,
-          "logprob": -16.453125,
+          "logprob": -21.59375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.09375,
+          "logprob": -20.515625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.5625,
+          "logprob": -19.171875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.15625,
+          "logprob": -19.296875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.171875,
+          "logprob": -16.71875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.671875,
+          "logprob": -20.46875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.859375,
+          "logprob": -21.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.40625,
           "text": "<image>"
         },
         {
@@ -6324,37 +4020,77 @@
         },
         {
           "id": 32001,
-          "logprob": -21.8125,
+          "logprob": -17.765625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.96875,
+          "logprob": -20.328125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.046875,
+          "logprob": -15.2734375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.78125,
+          "logprob": -18.84375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.421875,
+          "logprob": -17.875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.21875,
+          "logprob": -15.578125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.515625,
+          "logprob": -18.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.15625,
           "text": "<image>"
         },
         {
@@ -6364,37 +4100,12 @@
         },
         {
           "id": 32001,
-          "logprob": -20.734375,
+          "logprob": -19.546875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.1875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.1875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.75,
+          "logprob": -14.1953125,
           "text": "<image>"
         },
         {
@@ -6404,182 +4115,42 @@
         },
         {
           "id": 32001,
-          "logprob": -18.765625,
+          "logprob": -18.1875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.453125,
+          "logprob": -17.421875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.890625,
+          "logprob": -20.421875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.015625,
+          "logprob": -20.0,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.90625,
+          "logprob": -20.359375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.953125,
+          "logprob": -18.03125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.46875,
+          "logprob": -17.203125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.1875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -19.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.890625,
+          "logprob": -16.84375,
           "text": "<image>"
         },
         {
@@ -6589,72 +4160,32 @@
         },
         {
           "id": 32001,
-          "logprob": -18.75,
+          "logprob": -15.71875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.90625,
+          "logprob": -18.203125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.375,
+          "logprob": -18.4375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.640625,
+          "logprob": -18.46875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.859375,
+          "logprob": -17.3125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -15.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -15.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -17.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -18.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -21.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -20.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32001,
-          "logprob": -16.25,
+          "logprob": -16.265625,
           "text": "<image>"
         },
         {
@@ -6664,37 +4195,47 @@
         },
         {
           "id": 32001,
-          "logprob": -21.59375,
+          "logprob": -14.734375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -22.421875,
+          "logprob": -20.6875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.9375,
+          "logprob": -20.21875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.671875,
+          "logprob": -18.359375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.890625,
+          "logprob": -18.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -3.2988281,
+          "text": "<fake_token_around_image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -25.75,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.921875,
+          "logprob": -18.421875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.5,
+          "logprob": -19.265625,
           "text": "<image>"
         },
         {
@@ -6704,42 +4245,77 @@
         },
         {
           "id": 32001,
-          "logprob": -22.1875,
+          "logprob": -17.203125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.734375,
+          "logprob": -20.140625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.40625,
+          "logprob": -17.96875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.453125,
+          "logprob": -16.453125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.234375,
+          "logprob": -19.65625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -22.28125,
+          "logprob": -18.203125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.515625,
+          "logprob": -16.921875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.4296875,
+          "logprob": -15.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.984375,
           "text": "<image>"
         },
         {
@@ -6749,27 +4325,187 @@
         },
         {
           "id": 32001,
-          "logprob": -14.6484375,
+          "logprob": -19.890625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.46875,
+          "logprob": -20.421875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.875,
+          "logprob": -19.34375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.859375,
+          "logprob": -20.140625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.078125,
+          "logprob": -19.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -15.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -15.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -15.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -15.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.703125,
           "text": "<image>"
         },
         {
@@ -6777,11 +4513,581 @@
           "logprob": -16.4375,
           "text": "<image>"
         },
+        {
+          "id": 32001,
+          "logprob": -19.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -2.7207031,
+          "text": "<fake_token_around_image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -23.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -22.203125,
+          "text": "<image>"
+        },
         {
           "id": 32001,
           "logprob": -21.015625,
           "text": "<image>"
         },
+        {
+          "id": 32001,
+          "logprob": -18.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -15.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -15.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -15.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -3.0917969,
+          "text": "<fake_token_around_image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -25.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -15.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -15.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -22.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -22.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -22.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -15.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -14.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.046875,
+          "text": "<image>"
+        },
         {
           "id": 32001,
           "logprob": -21.234375,
@@ -6794,22 +5100,22 @@
         },
         {
           "id": 32001,
-          "logprob": -21.484375,
+          "logprob": -21.515625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -20.015625,
+          "logprob": -20.0,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -18.84375,
+          "logprob": -18.78125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.421875,
+          "logprob": -16.375,
           "text": "<image>"
         },
         {
@@ -6819,57 +5125,57 @@
         },
         {
           "id": 32001,
-          "logprob": -16.65625,
+          "logprob": -16.703125,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -13.640625,
+          "logprob": -13.625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.4140625,
+          "logprob": -15.375,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.546875,
+          "logprob": -17.515625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -21.859375,
+          "logprob": -21.921875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.65625,
+          "logprob": -15.640625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.484375,
+          "logprob": -16.46875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -16.359375,
+          "logprob": -16.421875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -19.9375,
+          "logprob": -19.890625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.875,
+          "logprob": -17.890625,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -17.453125,
+          "logprob": -17.40625,
           "text": "<image>"
         },
         {
@@ -6879,72 +5185,72 @@
         },
         {
           "id": 32001,
-          "logprob": -19.171875,
+          "logprob": -19.1875,
           "text": "<image>"
         },
         {
           "id": 32001,
-          "logprob": -15.9921875,
+          "logprob": -15.9609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -2.0429688,
+          "logprob": -2.0332031,
           "text": "<fake_token_around_image>"
         },
         {
           "id": 12018,
-          "logprob": -12.03125,
+          "logprob": -12.078125,
           "text": "Write"
         },
         {
           "id": 528,
-          "logprob": -10.25,
+          "logprob": -10.109375,
           "text": "me"
         },
         {
           "id": 264,
-          "logprob": -0.10437012,
+          "logprob": -0.103515625,
           "text": "a"
         },
         {
           "id": 2485,
-          "logprob": -4.578125,
+          "logprob": -4.5664062,
           "text": "short"
         },
         {
           "id": 2838,
-          "logprob": -0.22924805,
+          "logprob": -0.23864746,
           "text": "story"
         },
         {
           "id": 32002,
-          "logprob": -10.84375,
+          "logprob": -10.9609375,
           "text": "<end_of_utterance>"
         },
         {
           "id": 259,
-          "logprob": -20.171875,
+          "logprob": -20.34375,
           "text": " "
         },
         {
           "id": 13,
-          "logprob": -8.765625,
+          "logprob": -8.5546875,
           "text": "\n"
         },
         {
           "id": 7226,
-          "logprob": -10.4140625,
+          "logprob": -10.484375,
           "text": "Ass"
         },
         {
           "id": 11143,
-          "logprob": -13.640625,
+          "logprob": -13.6015625,
           "text": "istant"
         },
         {
           "id": 28747,
-          "logprob": -0.005744934,
+          "logprob": -0.008308411,
           "text": ":"
         }
       ],
@@ -6952,61 +5258,1835 @@
       "tokens": [
         {
           "id": 330,
-          "logprob": -0.12976074,
+          "logprob": -0.09448242,
           "special": false,
           "text": " A"
         },
         {
           "id": 13088,
-          "logprob": -0.66308594,
+          "logprob": -0.6743164,
           "special": false,
           "text": " chicken"
         },
         {
           "id": 349,
-          "logprob": -0.29541016,
+          "logprob": -0.31201172,
           "special": false,
           "text": " is"
         },
         {
           "id": 6398,
-          "logprob": -0.05996704,
+          "logprob": -0.051635742,
           "special": false,
           "text": " sitting"
         },
         {
           "id": 356,
-          "logprob": -0.27075195,
+          "logprob": -0.34033203,
           "special": false,
           "text": " on"
         },
         {
           "id": 264,
-          "logprob": -0.14160156,
+          "logprob": -0.1194458,
           "special": false,
           "text": " a"
         },
         {
           "id": 17972,
-          "logprob": -0.040863037,
+          "logprob": -0.032562256,
           "special": false,
           "text": " pile"
         },
         {
           "id": 302,
-          "logprob": -0.00027036667,
+          "logprob": -0.00018787384,
           "special": false,
           "text": " of"
         },
         {
           "id": 2445,
-          "logprob": -0.093322754,
+          "logprob": -0.07122803,
           "special": false,
           "text": " money"
         },
         {
           "id": 28723,
-          "logprob": -0.006931305,
+          "logprob": -0.0041007996,
+          "special": false,
+          "text": "."
+        }
+      ],
+      "top_tokens": null
+    },
+    "generated_text": " A chicken is sitting on a pile of money."
+  },
+  {
+    "details": {
+      "best_of_sequences": null,
+      "finish_reason": "length",
+      "generated_tokens": 10,
+      "prefill": [
+        {
+          "id": 1,
+          "logprob": null,
+          "text": "<s>"
+        },
+        {
+          "id": 1247,
+          "logprob": -5.2382812,
+          "text": "User"
+        },
+        {
+          "id": 28747,
+          "logprob": -6.9492188,
+          "text": ":"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.234375,
+          "text": "<fake_token_around_image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -23.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -15.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -22.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -23.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -23.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -14.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -23.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -23.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -23.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -3.015625,
+          "text": "<fake_token_around_image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -22.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -15.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -15.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -15.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -14.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -15.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -14.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -3.2988281,
+          "text": "<fake_token_around_image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -25.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -15.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -15.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -15.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -15.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -15.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -2.7207031,
+          "text": "<fake_token_around_image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -23.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -22.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -15.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -15.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -15.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -3.0917969,
+          "text": "<fake_token_around_image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -25.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -15.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -15.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -22.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -22.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -22.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -15.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -14.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -18.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -13.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -15.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -21.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -15.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -16.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -17.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -20.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -19.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32001,
+          "logprob": -15.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -2.0332031,
+          "text": "<fake_token_around_image>"
+        },
+        {
+          "id": 12018,
+          "logprob": -12.078125,
+          "text": "Write"
+        },
+        {
+          "id": 528,
+          "logprob": -10.109375,
+          "text": "me"
+        },
+        {
+          "id": 264,
+          "logprob": -0.103515625,
+          "text": "a"
+        },
+        {
+          "id": 2485,
+          "logprob": -4.5664062,
+          "text": "short"
+        },
+        {
+          "id": 2838,
+          "logprob": -0.23864746,
+          "text": "story"
+        },
+        {
+          "id": 32002,
+          "logprob": -10.9609375,
+          "text": "<end_of_utterance>"
+        },
+        {
+          "id": 259,
+          "logprob": -20.34375,
+          "text": " "
+        },
+        {
+          "id": 13,
+          "logprob": -8.5546875,
+          "text": "\n"
+        },
+        {
+          "id": 7226,
+          "logprob": -10.484375,
+          "text": "Ass"
+        },
+        {
+          "id": 11143,
+          "logprob": -13.6015625,
+          "text": "istant"
+        },
+        {
+          "id": 28747,
+          "logprob": -0.008308411,
+          "text": ":"
+        }
+      ],
+      "seed": null,
+      "tokens": [
+        {
+          "id": 330,
+          "logprob": -0.09448242,
+          "special": false,
+          "text": " A"
+        },
+        {
+          "id": 13088,
+          "logprob": -0.6743164,
+          "special": false,
+          "text": " chicken"
+        },
+        {
+          "id": 349,
+          "logprob": -0.31201172,
+          "special": false,
+          "text": " is"
+        },
+        {
+          "id": 6398,
+          "logprob": -0.051635742,
+          "special": false,
+          "text": " sitting"
+        },
+        {
+          "id": 356,
+          "logprob": -0.34033203,
+          "special": false,
+          "text": " on"
+        },
+        {
+          "id": 264,
+          "logprob": -0.1194458,
+          "special": false,
+          "text": " a"
+        },
+        {
+          "id": 17972,
+          "logprob": -0.032562256,
+          "special": false,
+          "text": " pile"
+        },
+        {
+          "id": 302,
+          "logprob": -0.00018763542,
+          "special": false,
+          "text": " of"
+        },
+        {
+          "id": 2445,
+          "logprob": -0.07122803,
+          "special": false,
+          "text": " money"
+        },
+        {
+          "id": 28723,
+          "logprob": -0.0041007996,
           "special": false,
           "text": "."
         }
diff --git a/integration-tests/models/__snapshots__/test_idefics2/test_flash_idefics2_next_simple.json b/integration-tests/models/__snapshots__/test_idefics2/test_flash_idefics2_next_simple.json
index a3b18d0a..da2ac897 100644
--- a/integration-tests/models/__snapshots__/test_idefics2/test_flash_idefics2_next_simple.json
+++ b/integration-tests/models/__snapshots__/test_idefics2/test_flash_idefics2_next_simple.json
@@ -8,61 +8,61 @@
     "tokens": [
       {
         "id": 330,
-        "logprob": -0.13000488,
+        "logprob": -0.08660889,
         "special": false,
         "text": " A"
       },
       {
         "id": 13088,
-        "logprob": -0.6713867,
+        "logprob": -0.7089844,
         "special": false,
         "text": " chicken"
       },
       {
         "id": 349,
-        "logprob": -0.2980957,
+        "logprob": -0.32885742,
         "special": false,
         "text": " is"
       },
       {
         "id": 6398,
-        "logprob": -0.060638428,
+        "logprob": -0.05126953,
         "special": false,
         "text": " sitting"
       },
       {
         "id": 356,
-        "logprob": -0.27319336,
+        "logprob": -0.35229492,
         "special": false,
         "text": " on"
       },
       {
         "id": 264,
-        "logprob": -0.140625,
+        "logprob": -0.12561035,
         "special": false,
         "text": " a"
       },
       {
         "id": 17972,
-        "logprob": -0.040405273,
+        "logprob": -0.038085938,
         "special": false,
         "text": " pile"
       },
       {
         "id": 302,
-        "logprob": -0.0002708435,
+        "logprob": -0.00018656254,
         "special": false,
         "text": " of"
       },
       {
         "id": 2445,
-        "logprob": -0.095336914,
+        "logprob": -0.07293701,
         "special": false,
         "text": " money"
       },
       {
         "id": 28723,
-        "logprob": -0.0068359375,
+        "logprob": -0.004852295,
         "special": false,
         "text": "."
       }
diff --git a/integration-tests/models/__snapshots__/test_idefics2/test_flash_idefics2_two_images.json b/integration-tests/models/__snapshots__/test_idefics2/test_flash_idefics2_two_images.json
new file mode 100644
index 00000000..44ccea71
--- /dev/null
+++ b/integration-tests/models/__snapshots__/test_idefics2/test_flash_idefics2_two_images.json
@@ -0,0 +1,127 @@
+{
+  "details": {
+    "best_of_sequences": null,
+    "finish_reason": "eos_token",
+    "generated_tokens": 19,
+    "prefill": [],
+    "seed": null,
+    "tokens": [
+      {
+        "id": 415,
+        "logprob": -0.03665161,
+        "special": false,
+        "text": " The"
+      },
+      {
+        "id": 12072,
+        "logprob": -0.13549805,
+        "special": false,
+        "text": " cow"
+      },
+      {
+        "id": 349,
+        "logprob": -0.05819702,
+        "special": false,
+        "text": " is"
+      },
+      {
+        "id": 6328,
+        "logprob": -0.6826172,
+        "special": false,
+        "text": " standing"
+      },
+      {
+        "id": 356,
+        "logprob": -0.1607666,
+        "special": false,
+        "text": " on"
+      },
+      {
+        "id": 272,
+        "logprob": -0.5073242,
+        "special": false,
+        "text": " the"
+      },
+      {
+        "id": 10305,
+        "logprob": -0.016418457,
+        "special": false,
+        "text": " beach"
+      },
+      {
+        "id": 304,
+        "logprob": -1.3916016,
+        "special": false,
+        "text": " and"
+      },
+      {
+        "id": 272,
+        "logprob": -0.020217896,
+        "special": false,
+        "text": " the"
+      },
+      {
+        "id": 13088,
+        "logprob": -0.0028133392,
+        "special": false,
+        "text": " chicken"
+      },
+      {
+        "id": 349,
+        "logprob": -0.003145218,
+        "special": false,
+        "text": " is"
+      },
+      {
+        "id": 6398,
+        "logprob": -0.37060547,
+        "special": false,
+        "text": " sitting"
+      },
+      {
+        "id": 356,
+        "logprob": -0.034851074,
+        "special": false,
+        "text": " on"
+      },
+      {
+        "id": 264,
+        "logprob": -0.2878418,
+        "special": false,
+        "text": " a"
+      },
+      {
+        "id": 17972,
+        "logprob": -0.046051025,
+        "special": false,
+        "text": " pile"
+      },
+      {
+        "id": 302,
+        "logprob": -0.00028848648,
+        "special": false,
+        "text": " of"
+      },
+      {
+        "id": 2445,
+        "logprob": -0.025772095,
+        "special": false,
+        "text": " money"
+      },
+      {
+        "id": 28723,
+        "logprob": -0.018127441,
+        "special": false,
+        "text": "."
+      },
+      {
+        "id": 32002,
+        "logprob": -0.00019824505,
+        "special": true,
+        "text": "<end_of_utterance>"
+      }
+    ],
+    "top_tokens": null
+  },
+  "generated_text": " The cow is standing on the beach and the chicken is sitting on a pile of money."
+}
diff --git a/integration-tests/models/__snapshots__/test_llava_next/test_flash_llava_next_load.json b/integration-tests/models/__snapshots__/test_llava_next/test_flash_llava_next_load.json
index 2007c0f2..50ce2e31 100644
--- a/integration-tests/models/__snapshots__/test_llava_next/test_flash_llava_next_load.json
+++ b/integration-tests/models/__snapshots__/test_llava_next/test_flash_llava_next_load.json
@@ -12,7 +12,7 @@
         },
         {
           "id": 1247,
-          "logprob": -2.3886719,
+          "logprob": -2.390625,
           "text": "User"
         },
         {
@@ -27,12 +27,12 @@
         },
         {
           "id": 32000,
-          "logprob": -10.671875,
+          "logprob": -10.6953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.7109375,
+          "logprob": -15.828125,
           "text": "<image>"
         },
         {
@@ -42,82 +42,82 @@
         },
         {
           "id": 32000,
-          "logprob": -10.0234375,
+          "logprob": -10.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.1328125,
+          "logprob": -10.140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.421875,
+          "logprob": -10.3125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.90625,
+          "logprob": -12.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.59375,
+          "logprob": -15.4921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.828125,
+          "logprob": -13.8671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.390625,
+          "logprob": -11.3671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.1171875,
+          "logprob": -10.1015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.1640625,
+          "logprob": -10.1796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.234375,
+          "logprob": -10.2421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.3984375,
+          "logprob": -10.4609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.015625,
+          "logprob": -14.34375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0859375,
+          "logprob": -13.1796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2734375,
+          "logprob": -13.328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.359375,
+          "logprob": -14.3515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0390625,
+          "logprob": -11.0546875,
           "text": "<image>"
         },
         {
@@ -127,12 +127,12 @@
         },
         {
           "id": 32000,
-          "logprob": -10.5234375,
+          "logprob": -10.515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.4765625,
+          "logprob": -10.4453125,
           "text": "<image>"
         },
         {
@@ -142,22 +142,22 @@
         },
         {
           "id": 32000,
-          "logprob": -13.6171875,
+          "logprob": -13.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.359375,
+          "logprob": -11.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.8359375,
+          "logprob": -10.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -17.34375,
+          "logprob": -17.234375,
           "text": "<image>"
         },
         {
@@ -172,12 +172,12 @@
         },
         {
           "id": 32000,
-          "logprob": -10.640625,
+          "logprob": -10.6484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -18.390625,
+          "logprob": -17.984375,
           "text": "<image>"
         },
         {
@@ -187,17 +187,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.5625,
+          "logprob": -11.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -9.875,
+          "logprob": -9.8671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.7734375,
+          "logprob": -10.7578125,
           "text": "<image>"
         },
         {
@@ -207,32 +207,32 @@
         },
         {
           "id": 32000,
-          "logprob": -10.96875,
+          "logprob": -10.984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.609375,
+          "logprob": -10.625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.09375,
+          "logprob": -11.109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.5078125,
+          "logprob": -10.5,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4453125,
+          "logprob": -11.4609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.59375,
+          "logprob": -13.09375,
           "text": "<image>"
         },
         {
@@ -242,132 +242,87 @@
         },
         {
           "id": 32000,
-          "logprob": -10.5625,
+          "logprob": -10.5390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.640625,
+          "logprob": -10.625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9765625,
+          "logprob": -10.9296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.765625,
+          "logprob": -11.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3671875,
+          "logprob": -12.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0234375,
+          "logprob": -12.125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.59375,
+          "logprob": -10.578125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.7421875,
+          "logprob": -10.7265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0625,
+          "logprob": -11.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.3828125,
+          "logprob": -10.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.171875,
+          "logprob": -11.1796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0234375,
+          "logprob": -11.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -18.40625,
+          "logprob": -18.703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9921875,
+          "logprob": -11.015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.7109375,
+          "logprob": -15.6796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.15625,
+          "logprob": -12.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.140625,
+          "logprob": -10.4140625,
           "text": "<image>"
         },
         {
@@ -377,12 +332,42 @@
         },
         {
           "id": 32000,
-          "logprob": -10.7890625,
+          "logprob": -11.8046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4296875,
+          "logprob": -10.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.140625,
           "text": "<image>"
         },
         {
@@ -392,12 +377,27 @@
         },
         {
           "id": 32000,
-          "logprob": -10.796875,
+          "logprob": -10.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1640625,
+          "logprob": -11.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.15625,
           "text": "<image>"
         },
         {
@@ -407,27 +407,27 @@
         },
         {
           "id": 32000,
-          "logprob": -15.4453125,
+          "logprob": -15.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.2109375,
+          "logprob": -10.2265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.09375,
+          "logprob": -11.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.6796875,
+          "logprob": -14.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.3671875,
+          "logprob": -10.375,
           "text": "<image>"
         },
         {
@@ -442,22 +442,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.484375,
+          "logprob": -12.4921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.09375,
+          "logprob": -12.1015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.96875,
+          "logprob": -11.0859375,
           "text": "<image>"
         },
         {
@@ -465,6 +460,11 @@
           "logprob": -10.9765625,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -10.921875,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -11.0234375,
@@ -480,11 +480,6 @@
           "logprob": -11.1484375,
           "text": "<image>"
         },
-        {
-          "id": 32000,
-          "logprob": -10.703125,
-          "text": "<image>"
-        },
         {
           "id": 32000,
           "logprob": -10.71875,
@@ -492,17 +487,22 @@
         },
         {
           "id": 32000,
-          "logprob": -11.6875,
+          "logprob": -10.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.8984375,
+          "logprob": -11.6796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2890625,
+          "logprob": -10.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.375,
           "text": "<image>"
         },
         {
@@ -512,12 +512,12 @@
         },
         {
           "id": 32000,
-          "logprob": -10.640625,
+          "logprob": -10.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.7109375,
+          "logprob": -10.7265625,
           "text": "<image>"
         },
         {
@@ -532,17 +532,17 @@
         },
         {
           "id": 32000,
-          "logprob": -10.6875,
+          "logprob": -10.6796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.84375,
+          "logprob": -11.828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5078125,
+          "logprob": -11.5390625,
           "text": "<image>"
         },
         {
@@ -552,22 +552,22 @@
         },
         {
           "id": 32000,
-          "logprob": -10.9609375,
+          "logprob": -10.9765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4921875,
+          "logprob": -12.5,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5546875,
+          "logprob": -11.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.2265625,
+          "logprob": -11.2421875,
           "text": "<image>"
         },
         {
@@ -577,7 +577,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.21875,
+          "logprob": -11.2265625,
           "text": "<image>"
         },
         {
@@ -587,52 +587,52 @@
         },
         {
           "id": 32000,
-          "logprob": -11.0,
+          "logprob": -10.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0234375,
+          "logprob": -12.046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.09375,
+          "logprob": -11.109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.046875,
+          "logprob": -14.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.921875,
+          "logprob": -10.8984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9609375,
+          "logprob": -10.953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9140625,
+          "logprob": -10.9296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8125,
+          "logprob": -11.78125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.140625,
+          "logprob": -15.1484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9609375,
+          "logprob": -10.96875,
           "text": "<image>"
         },
         {
@@ -642,72 +642,72 @@
         },
         {
           "id": 32000,
-          "logprob": -10.8828125,
+          "logprob": -10.890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9609375,
+          "logprob": -10.9453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.46875,
+          "logprob": -11.453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5234375,
+          "logprob": -13.09375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.328125,
+          "logprob": -12.484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.375,
+          "logprob": -11.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3515625,
+          "logprob": -12.3359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.578125,
+          "logprob": -11.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5078125,
+          "logprob": -12.5,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.640625,
+          "logprob": -12.6328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1953125,
+          "logprob": -11.203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9921875,
+          "logprob": -11.9765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.921875,
+          "logprob": -10.9296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.7578125,
+          "logprob": -10.7421875,
           "text": "<image>"
         },
         {
@@ -717,12 +717,12 @@
         },
         {
           "id": 32000,
-          "logprob": -15.015625,
+          "logprob": -15.0234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7734375,
+          "logprob": -12.75,
           "text": "<image>"
         },
         {
@@ -737,27 +737,27 @@
         },
         {
           "id": 32000,
-          "logprob": -11.2890625,
+          "logprob": -11.265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.34375,
+          "logprob": -13.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.6953125,
+          "logprob": -10.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.59375,
+          "logprob": -10.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6015625,
+          "logprob": -11.7578125,
           "text": "<image>"
         },
         {
@@ -772,7 +772,7 @@
         },
         {
           "id": 32000,
-          "logprob": -10.5390625,
+          "logprob": -10.5859375,
           "text": "<image>"
         },
         {
@@ -782,12 +782,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.9765625,
+          "logprob": -17.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4609375,
+          "logprob": -11.4765625,
           "text": "<image>"
         },
         {
@@ -797,12 +797,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.015625,
+          "logprob": -12.0078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.84375,
+          "logprob": -12.8359375,
           "text": "<image>"
         },
         {
@@ -812,12 +812,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.4375,
+          "logprob": -12.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.671875,
+          "logprob": -12.6875,
           "text": "<image>"
         },
         {
@@ -827,57 +827,27 @@
         },
         {
           "id": 32000,
-          "logprob": -12.875,
+          "logprob": -12.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2578125,
+          "logprob": -12.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.359375,
+          "logprob": -10.3359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.375,
+          "logprob": -13.46875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.7421875,
+          "logprob": -11.8671875,
           "text": "<image>"
         },
         {
@@ -887,7 +857,32 @@
         },
         {
           "id": 32000,
-          "logprob": -11.0390625,
+          "logprob": -11.0078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8203125,
           "text": "<image>"
         },
         {
@@ -897,17 +892,22 @@
         },
         {
           "id": 32000,
-          "logprob": -10.6171875,
+          "logprob": -10.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2421875,
+          "logprob": -10.6328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.8359375,
+          "logprob": -12.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8203125,
           "text": "<image>"
         },
         {
@@ -927,127 +927,62 @@
         },
         {
           "id": 32000,
-          "logprob": -11.0,
+          "logprob": -10.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7734375,
+          "logprob": -12.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2109375,
+          "logprob": -12.21875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4296875,
+          "logprob": -11.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2421875,
+          "logprob": -12.828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3046875,
+          "logprob": -11.328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.2890625,
+          "logprob": -10.296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.8203125,
+          "logprob": -10.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9140625,
+          "logprob": -11.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.2421875,
+          "logprob": -11.21875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.234375,
+          "logprob": -11.2109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4375,
+          "logprob": -11.5078125,
           "text": "<image>"
         },
         {
@@ -1057,27 +992,12 @@
         },
         {
           "id": 32000,
-          "logprob": -14.203125,
+          "logprob": -10.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.59375,
+          "logprob": -10.3515625,
           "text": "<image>"
         },
         {
@@ -1087,102 +1007,22 @@
         },
         {
           "id": 32000,
-          "logprob": -10.90625,
+          "logprob": -10.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.2578125,
+          "logprob": -10.828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.921875,
+          "logprob": -11.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.890625,
+          "logprob": -11.9375,
           "text": "<image>"
         },
         {
@@ -1192,112 +1032,32 @@
         },
         {
           "id": 32000,
-          "logprob": -10.9609375,
+          "logprob": -12.3046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.7890625,
+          "logprob": -11.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4453125,
+          "logprob": -12.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.390625,
+          "logprob": -13.3984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1640625,
+          "logprob": -11.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0703125,
+          "logprob": -14.1875,
           "text": "<image>"
         },
         {
@@ -1307,72 +1067,162 @@
         },
         {
           "id": 32000,
-          "logprob": -12.859375,
+          "logprob": -12.421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1875,
+          "logprob": -10.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.171875,
+          "logprob": -10.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6640625,
+          "logprob": -10.6015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1953125,
+          "logprob": -10.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1328125,
+          "logprob": -11.2265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1875,
+          "logprob": -10.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9453125,
+          "logprob": -10.9765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8515625,
+          "logprob": -12.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.203125,
+          "logprob": -10.8671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1796875,
+          "logprob": -10.8515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9609375,
+          "logprob": -11.8984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.703125,
+          "logprob": -10.7421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8515625,
+          "logprob": -13.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1796875,
           "text": "<image>"
         },
         {
@@ -1380,11 +1230,161 @@
           "logprob": -11.75,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -10.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.640625,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -11.8359375,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -11.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.828125,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -12.1953125,
@@ -1392,32 +1392,32 @@
         },
         {
           "id": 32000,
-          "logprob": -12.5078125,
+          "logprob": -12.4921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.546875,
+          "logprob": -11.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.078125,
+          "logprob": -12.0859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.2421875,
+          "logprob": -11.265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6640625,
+          "logprob": -11.6953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2109375,
+          "logprob": -12.21875,
           "text": "<image>"
         },
         {
@@ -1437,27 +1437,7 @@
         },
         {
           "id": 32000,
-          "logprob": -15.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3046875,
+          "logprob": -15.03125,
           "text": "<image>"
         },
         {
@@ -1467,32 +1447,52 @@
         },
         {
           "id": 32000,
-          "logprob": -12.2265625,
+          "logprob": -11.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9453125,
+          "logprob": -13.1171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4921875,
+          "logprob": -12.296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.8828125,
+          "logprob": -12.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3125,
+          "logprob": -12.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8984375,
+          "logprob": -12.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9140625,
           "text": "<image>"
         },
         {
@@ -1502,12 +1502,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8125,
+          "logprob": -11.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.90625,
+          "logprob": -11.9140625,
           "text": "<image>"
         },
         {
@@ -1527,12 +1527,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.140625,
+          "logprob": -13.1484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9765625,
+          "logprob": -11.9921875,
           "text": "<image>"
         },
         {
@@ -1547,7 +1547,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.84375,
+          "logprob": -11.8359375,
           "text": "<image>"
         },
         {
@@ -1562,17 +1562,22 @@
         },
         {
           "id": 32000,
-          "logprob": -13.1953125,
+          "logprob": -13.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.875,
+          "logprob": -11.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.6015625,
+          "logprob": -13.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6484375,
           "text": "<image>"
         },
         {
@@ -1582,27 +1587,22 @@
         },
         {
           "id": 32000,
-          "logprob": -13.671875,
+          "logprob": -11.4609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5390625,
+          "logprob": -13.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.59375,
+          "logprob": -12.4375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5703125,
+          "logprob": -14.6328125,
           "text": "<image>"
         },
         {
@@ -1612,57 +1612,57 @@
         },
         {
           "id": 32000,
-          "logprob": -13.4140625,
+          "logprob": -13.3984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7890625,
+          "logprob": -11.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6328125,
+          "logprob": -11.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.4296875,
+          "logprob": -14.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.53125,
+          "logprob": -13.5390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.515625,
+          "logprob": -14.5078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7265625,
+          "logprob": -12.7109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.609375,
+          "logprob": -11.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.171875,
+          "logprob": -13.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.109375,
+          "logprob": -13.09375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8828125,
+          "logprob": -12.875,
           "text": "<image>"
         },
         {
@@ -1672,37 +1672,37 @@
         },
         {
           "id": 32000,
-          "logprob": -12.671875,
+          "logprob": -12.6484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.7109375,
+          "logprob": -13.6953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4296875,
+          "logprob": -12.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.296875,
+          "logprob": -12.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1796875,
+          "logprob": -13.109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2421875,
+          "logprob": -12.1796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.828125,
+          "logprob": -14.875,
           "text": "<image>"
         },
         {
@@ -1712,7 +1712,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.3359375,
+          "logprob": -11.34375,
           "text": "<image>"
         },
         {
@@ -1732,67 +1732,7 @@
         },
         {
           "id": 32000,
-          "logprob": -15.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0,
+          "logprob": -15.0859375,
           "text": "<image>"
         },
         {
@@ -1802,57 +1742,117 @@
         },
         {
           "id": 32000,
-          "logprob": -16.21875,
+          "logprob": -11.875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6015625,
+          "logprob": -13.25,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.796875,
+          "logprob": -13.5078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7265625,
+          "logprob": -13.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9453125,
+          "logprob": -14.265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9765625,
+          "logprob": -14.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.71875,
+          "logprob": -13.2421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.6953125,
+          "logprob": -12.28125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3359375,
+          "logprob": -13.8046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3203125,
+          "logprob": -12.703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6328125,
+          "logprob": -12.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.609375,
           "text": "<image>"
         },
         {
@@ -1862,17 +1862,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1875,
+          "logprob": -14.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.046875,
+          "logprob": -12.0234375,
           "text": "<image>"
         },
         {
@@ -1887,17 +1887,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.8828125,
+          "logprob": -12.890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.296875,
+          "logprob": -12.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2109375,
+          "logprob": -12.21875,
           "text": "<image>"
         },
         {
@@ -1907,27 +1907,27 @@
         },
         {
           "id": 32000,
-          "logprob": -15.1875,
+          "logprob": -15.1953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5390625,
+          "logprob": -11.546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.421875,
+          "logprob": -14.4375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2890625,
+          "logprob": -12.265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2265625,
+          "logprob": -12.21875,
           "text": "<image>"
         },
         {
@@ -1942,12 +1942,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.5859375,
+          "logprob": -13.5390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.859375,
+          "logprob": -11.8515625,
           "text": "<image>"
         },
         {
@@ -1957,12 +1957,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.1015625,
+          "logprob": -13.046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.9453125,
+          "logprob": -13.953125,
           "text": "<image>"
         },
         {
@@ -1977,47 +1977,47 @@
         },
         {
           "id": 32000,
-          "logprob": -12.734375,
+          "logprob": -12.7109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.203125,
+          "logprob": -12.1796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.59375,
+          "logprob": -12.578125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3984375,
+          "logprob": -11.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5,
+          "logprob": -13.4609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.765625,
+          "logprob": -11.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2265625,
+          "logprob": -13.21875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7578125,
+          "logprob": -11.75,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.3515625,
+          "logprob": -15.34375,
           "text": "<image>"
         },
         {
@@ -2032,22 +2032,42 @@
         },
         {
           "id": 32000,
-          "logprob": -13.3671875,
+          "logprob": -13.3515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.90625,
+          "logprob": -11.8828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5625,
+          "logprob": -12.578125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3203125,
+          "logprob": -13.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.125,
           "text": "<image>"
         },
         {
@@ -2057,27 +2077,7 @@
         },
         {
           "id": 32000,
-          "logprob": -10.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9921875,
+          "logprob": -11.984375,
           "text": "<image>"
         },
         {
@@ -2092,27 +2092,27 @@
         },
         {
           "id": 32000,
-          "logprob": -12.9765625,
+          "logprob": -12.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4921875,
+          "logprob": -12.484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0625,
+          "logprob": -12.046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1796875,
+          "logprob": -12.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3359375,
+          "logprob": -13.296875,
           "text": "<image>"
         },
         {
@@ -2122,32 +2122,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.375,
+          "logprob": -12.3671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.984375,
+          "logprob": -13.1328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9453125,
+          "logprob": -13.625,
           "text": "<image>"
         },
         {
@@ -2157,87 +2142,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.3203125,
+          "logprob": -11.75,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3125,
+          "logprob": -11.9140625,
           "text": "<image>"
         },
         {
@@ -2247,12 +2157,102 @@
         },
         {
           "id": 32000,
-          "logprob": -11.3984375,
+          "logprob": -12.3359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.84375,
+          "logprob": -12.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8359375,
           "text": "<image>"
         },
         {
@@ -2262,52 +2262,52 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6328125,
+          "logprob": -12.6171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.875,
+          "logprob": -13.84375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.109375,
+          "logprob": -12.09375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1171875,
+          "logprob": -12.140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4921875,
+          "logprob": -11.5078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.2890625,
+          "logprob": -11.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.15625,
+          "logprob": -12.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.59375,
+          "logprob": -13.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8046875,
+          "logprob": -11.7890625,
           "text": "<image>"
         },
         {
@@ -2317,22 +2317,22 @@
         },
         {
           "id": 32000,
-          "logprob": -13.1015625,
+          "logprob": -13.0625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2265625,
+          "logprob": -13.1875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2109375,
+          "logprob": -13.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4609375,
+          "logprob": -12.46875,
           "text": "<image>"
         },
         {
@@ -2342,12 +2342,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
+          "logprob": -12.6953125,
           "text": "<image>"
         },
         {
@@ -2355,6 +2350,11 @@
           "logprob": -11.6875,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -11.6953125,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -11.7109375,
@@ -2362,27 +2362,27 @@
         },
         {
           "id": 32000,
-          "logprob": -15.390625,
+          "logprob": -15.3359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.953125,
+          "logprob": -11.9453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.140625,
+          "logprob": -16.125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.4921875,
+          "logprob": -15.4609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.9296875,
+          "logprob": -13.921875,
           "text": "<image>"
         },
         {
@@ -2397,7 +2397,7 @@
         },
         {
           "id": 32000,
-          "logprob": -15.984375,
+          "logprob": -16.0,
           "text": "<image>"
         },
         {
@@ -2407,37 +2407,37 @@
         },
         {
           "id": 32000,
-          "logprob": -12.8671875,
+          "logprob": -12.828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4921875,
+          "logprob": -12.5078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7421875,
+          "logprob": -11.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1875,
+          "logprob": -14.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3515625,
+          "logprob": -11.34375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.71875,
+          "logprob": -11.7265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0625,
+          "logprob": -13.046875,
           "text": "<image>"
         },
         {
@@ -2447,12 +2447,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.421875,
+          "logprob": -12.4375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.2734375,
+          "logprob": -14.265625,
           "text": "<image>"
         },
         {
@@ -2462,77 +2462,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.390625,
+          "logprob": -12.6953125,
           "text": "<image>"
         },
         {
@@ -2542,27 +2472,97 @@
         },
         {
           "id": 32000,
-          "logprob": -13.078125,
+          "logprob": -12.984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4296875,
+          "logprob": -13.2890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.828125,
+          "logprob": -13.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8046875,
+          "logprob": -11.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6484375,
+          "logprob": -10.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.625,
           "text": "<image>"
         },
         {
@@ -2577,27 +2577,27 @@
         },
         {
           "id": 32000,
-          "logprob": -14.921875,
+          "logprob": -14.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.578125,
+          "logprob": -12.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.78125,
+          "logprob": -12.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3984375,
+          "logprob": -11.375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.0546875,
+          "logprob": -14.03125,
           "text": "<image>"
         },
         {
@@ -2617,12 +2617,12 @@
         },
         {
           "id": 32000,
-          "logprob": -14.5234375,
+          "logprob": -14.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.625,
+          "logprob": -12.609375,
           "text": "<image>"
         },
         {
@@ -2632,12 +2632,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.6015625,
+          "logprob": -13.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.28125,
+          "logprob": -13.2109375,
           "text": "<image>"
         },
         {
@@ -2657,17 +2657,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.0,
+          "logprob": -12.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6640625,
+          "logprob": -12.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.46875,
+          "logprob": -11.484375,
           "text": "<image>"
         },
         {
@@ -2677,12 +2677,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8828125,
+          "logprob": -11.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1015625,
+          "logprob": -13.078125,
           "text": "<image>"
         },
         {
@@ -2697,7 +2697,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.5234375,
+          "logprob": -11.546875,
           "text": "<image>"
         },
         {
@@ -2712,7 +2712,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.4921875,
+          "logprob": -12.5,
           "text": "<image>"
         },
         {
@@ -2727,122 +2727,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.7890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1484375,
+          "logprob": -12.078125,
           "text": "<image>"
         },
         {
@@ -2852,132 +2737,102 @@
         },
         {
           "id": 32000,
-          "logprob": -13.3828125,
+          "logprob": -13.2265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3203125,
+          "logprob": -14.8046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.34375,
+          "logprob": -12.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -11.8671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.140625,
+          "logprob": -13.1484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9921875,
+          "logprob": -12.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.15625,
+          "logprob": -10.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.84375,
+          "logprob": -14.6875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.21875,
+          "logprob": -15.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.46875,
+          "logprob": -15.453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1484375,
+          "logprob": -11.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.515625,
+          "logprob": -14.484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.84375,
+          "logprob": -12.4375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.015625,
+          "logprob": -16.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.28125,
+          "logprob": -14.875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6015625,
+          "logprob": -16.109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3984375,
+          "logprob": -13.2890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.375,
+          "logprob": -12.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5625,
+          "logprob": -11.8359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -9.9375,
+          "logprob": -12.9921875,
           "text": "<image>"
         },
         {
@@ -2987,17 +2842,162 @@
         },
         {
           "id": 32000,
-          "logprob": -12.203125,
+          "logprob": -12.1484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.2890625,
+          "logprob": -13.84375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1953125,
+          "logprob": -13.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -9.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.15625,
           "text": "<image>"
         },
         {
@@ -3010,21 +3010,6 @@
           "logprob": -12.28125,
           "text": "<image>"
         },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
         {
           "id": 32000,
           "logprob": -11.8984375,
@@ -3032,52 +3017,67 @@
         },
         {
           "id": 32000,
-          "logprob": -10.5859375,
+          "logprob": -14.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9921875,
+          "logprob": -14.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9921875,
+          "logprob": -11.8671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1015625,
+          "logprob": -10.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.5390625,
+          "logprob": -13.0,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.2578125,
+          "logprob": -12.8984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1171875,
+          "logprob": -12.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2421875,
+          "logprob": -15.765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5,
+          "logprob": -15.203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7265625,
+          "logprob": -14.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8359375,
           "text": "<image>"
         },
         {
@@ -3087,7 +3087,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.0,
+          "logprob": -11.0078125,
           "text": "<image>"
         },
         {
@@ -3097,22 +3097,22 @@
         },
         {
           "id": 32000,
-          "logprob": -13.28125,
+          "logprob": -13.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7734375,
+          "logprob": -11.7578125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0625,
+          "logprob": -13.09375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.671875,
+          "logprob": -13.6796875,
           "text": "<image>"
         },
         {
@@ -3122,7 +3122,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.3828125,
+          "logprob": -12.421875,
           "text": "<image>"
         },
         {
@@ -3132,22 +3132,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.25,
+          "logprob": -12.2890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.9453125,
+          "logprob": -11.921875,
           "text": "<image>"
         },
         {
@@ -3157,7 +3147,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.9375,
+          "logprob": -15.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
           "text": "<image>"
         },
         {
@@ -3167,37 +3167,37 @@
         },
         {
           "id": 32000,
-          "logprob": -11.3203125,
+          "logprob": -11.328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4921875,
+          "logprob": -11.5,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.359375,
+          "logprob": -11.3671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3359375,
+          "logprob": -11.3515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0546875,
+          "logprob": -12.0625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.359375,
+          "logprob": -12.3359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -9.6953125,
+          "logprob": -9.7109375,
           "text": "<image>"
         },
         {
@@ -3207,27 +3207,27 @@
         },
         {
           "id": 32000,
-          "logprob": -14.3203125,
+          "logprob": -14.2421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9609375,
+          "logprob": -11.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0859375,
+          "logprob": -12.09375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2109375,
+          "logprob": -12.1796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4921875,
+          "logprob": -11.5234375,
           "text": "<image>"
         },
         {
@@ -3242,7 +3242,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.0546875,
           "text": "<image>"
         },
         {
@@ -3257,22 +3257,22 @@
         },
         {
           "id": 32000,
-          "logprob": -11.9921875,
+          "logprob": -12.0,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2265625,
+          "logprob": -12.1953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9921875,
+          "logprob": -12.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6796875,
+          "logprob": -12.7109375,
           "text": "<image>"
         },
         {
@@ -3282,7 +3282,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.5703125,
+          "logprob": -13.5546875,
           "text": "<image>"
         },
         {
@@ -3297,17 +3297,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.9375,
+          "logprob": -11.953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.9453125,
+          "logprob": -13.8046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.984375,
+          "logprob": -11.9921875,
           "text": "<image>"
         },
         {
@@ -3317,17 +3317,17 @@
         },
         {
           "id": 32000,
-          "logprob": -10.03125,
+          "logprob": -10.015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7265625,
+          "logprob": -11.71875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.7890625,
+          "logprob": -13.8125,
           "text": "<image>"
         },
         {
@@ -3337,42 +3337,42 @@
         },
         {
           "id": 32000,
-          "logprob": -12.671875,
+          "logprob": -12.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.59375,
+          "logprob": -12.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1171875,
+          "logprob": -12.1015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4609375,
+          "logprob": -12.453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3046875,
+          "logprob": -12.2890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4765625,
+          "logprob": -12.46875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6328125,
+          "logprob": -12.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.03125,
+          "logprob": -12.015625,
           "text": "<image>"
         },
         {
@@ -3382,132 +3382,32 @@
         },
         {
           "id": 32000,
-          "logprob": -14.5078125,
+          "logprob": -14.421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2265625,
+          "logprob": -13.25,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.546875,
+          "logprob": -16.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.4765625,
+          "logprob": -14.484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.71875,
+          "logprob": -13.7421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.7890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.671875,
+          "logprob": -13.46875,
           "text": "<image>"
         },
         {
@@ -3517,57 +3417,97 @@
         },
         {
           "id": 32000,
-          "logprob": -13.40625,
+          "logprob": -12.8671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.265625,
+          "logprob": -12.4921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.015625,
+          "logprob": -13.2265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5390625,
+          "logprob": -13.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.0234375,
+          "logprob": -12.4375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1796875,
+          "logprob": -10.765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6875,
+          "logprob": -13.109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.9453125,
+          "logprob": -12.8515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6875,
+          "logprob": -14.7421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.78125,
+          "logprob": -12.84375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.9609375,
+          "logprob": -12.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6484375,
           "text": "<image>"
         },
         {
@@ -3577,27 +3517,87 @@
         },
         {
           "id": 32000,
-          "logprob": -16.140625,
+          "logprob": -13.4296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4609375,
+          "logprob": -13.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.7265625,
+          "logprob": -12.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.609375,
+          "logprob": -11.5234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.234375,
+          "logprob": -15.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.25,
           "text": "<image>"
         },
         {
@@ -3607,7 +3607,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.4921875,
+          "logprob": -13.5,
           "text": "<image>"
         },
         {
@@ -3632,12 +3632,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.15625,
+          "logprob": -13.1484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -17.046875,
+          "logprob": -17.0625,
           "text": "<image>"
         },
         {
@@ -3647,12 +3647,12 @@
         },
         {
           "id": 32000,
-          "logprob": -14.265625,
+          "logprob": -14.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6328125,
+          "logprob": -12.625,
           "text": "<image>"
         },
         {
@@ -3662,92 +3662,37 @@
         },
         {
           "id": 32000,
-          "logprob": -13.9375,
+          "logprob": -13.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0390625,
+          "logprob": -13.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.46875,
+          "logprob": -10.5,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1640625,
+          "logprob": -13.1953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.59375,
+          "logprob": -13.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.390625,
+          "logprob": -13.3828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8125,
+          "logprob": -14.0625,
           "text": "<image>"
         },
         {
@@ -3755,6 +3700,61 @@
           "logprob": -13.5234375,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -14.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.53125,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -11.4140625,
@@ -3762,17 +3762,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.46875,
+          "logprob": -11.4609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.09375,
+          "logprob": -13.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.34375,
+          "logprob": -12.3359375,
           "text": "<image>"
         },
         {
@@ -3782,32 +3782,32 @@
         },
         {
           "id": 32000,
-          "logprob": -12.7578125,
+          "logprob": -12.75,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.8359375,
+          "logprob": -14.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0859375,
+          "logprob": -13.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1640625,
+          "logprob": -14.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0625,
+          "logprob": -13.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.5078125,
+          "logprob": -14.4921875,
           "text": "<image>"
         },
         {
@@ -3817,7 +3817,7 @@
         },
         {
           "id": 32000,
-          "logprob": -14.6953125,
+          "logprob": -14.703125,
           "text": "<image>"
         },
         {
@@ -3827,32 +3827,32 @@
         },
         {
           "id": 32000,
-          "logprob": -16.328125,
+          "logprob": -16.3125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.9921875,
+          "logprob": -14.9765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.8515625,
+          "logprob": -13.8046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.40625,
+          "logprob": -15.375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4296875,
+          "logprob": -13.3984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.5859375,
+          "logprob": -14.578125,
           "text": "<image>"
         },
         {
@@ -3867,7 +3867,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.3359375,
+          "logprob": -13.328125,
           "text": "<image>"
         },
         {
@@ -3882,37 +3882,52 @@
         },
         {
           "id": 32000,
-          "logprob": -16.890625,
+          "logprob": -16.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.328125,
+          "logprob": -13.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.78125,
+          "logprob": -13.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.34375,
+          "logprob": -13.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4921875,
+          "logprob": -13.4765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4296875,
+          "logprob": -13.390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.28125,
+          "logprob": -13.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5,
           "text": "<image>"
         },
         {
@@ -3922,7 +3937,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.75,
+          "logprob": -13.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6796875,
           "text": "<image>"
         },
         {
@@ -3932,32 +3957,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.328125,
+          "logprob": -13.3203125,
           "text": "<image>"
         },
         {
@@ -3967,47 +3967,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.640625,
+          "logprob": -13.6171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.15625,
+          "logprob": -15.140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0078125,
+          "logprob": -13.21875,
           "text": "<image>"
         },
         {
@@ -4017,27 +3987,57 @@
         },
         {
           "id": 32000,
-          "logprob": -11.625,
+          "logprob": -12.0234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8671875,
+          "logprob": -11.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.578125,
+          "logprob": -11.3125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8125,
+          "logprob": -10.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.9453125,
+          "logprob": -13.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.9765625,
           "text": "<image>"
         },
         {
@@ -4047,17 +4047,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.9765625,
+          "logprob": -12.984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.828125,
+          "logprob": -13.8359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7265625,
+          "logprob": -12.75,
           "text": "<image>"
         },
         {
@@ -4072,12 +4072,12 @@
         },
         {
           "id": 32000,
-          "logprob": -14.7578125,
+          "logprob": -14.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.5234375,
+          "logprob": -14.5078125,
           "text": "<image>"
         },
         {
@@ -4087,12 +4087,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.0390625,
+          "logprob": -13.046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4453125,
+          "logprob": -13.453125,
           "text": "<image>"
         },
         {
@@ -4107,17 +4107,17 @@
         },
         {
           "id": 32000,
-          "logprob": -14.0,
+          "logprob": -13.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5390625,
+          "logprob": -13.4921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0859375,
+          "logprob": -11.9765625,
           "text": "<image>"
         },
         {
@@ -4127,17 +4127,37 @@
         },
         {
           "id": 32000,
-          "logprob": -15.953125,
+          "logprob": -15.9609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.2265625,
+          "logprob": -14.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1484375,
+          "logprob": -13.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.640625,
           "text": "<image>"
         },
         {
@@ -4147,32 +4167,47 @@
         },
         {
           "id": 32000,
-          "logprob": -12.6875,
+          "logprob": -13.625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.796875,
+          "logprob": -12.890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.65625,
+          "logprob": -12.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6796875,
+          "logprob": -12.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.65625,
+          "logprob": -12.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.90625,
+          "logprob": -14.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0078125,
           "text": "<image>"
         },
         {
@@ -4182,52 +4217,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.6171875,
+          "logprob": -15.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7890625,
+          "logprob": -12.921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.375,
+          "logprob": -12.40625,
           "text": "<image>"
         },
         {
@@ -4242,27 +4242,27 @@
         },
         {
           "id": 32000,
-          "logprob": -11.375,
+          "logprob": -11.3671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0625,
+          "logprob": -11.0703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3046875,
+          "logprob": -11.28125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0390625,
+          "logprob": -11.046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3515625,
+          "logprob": -13.3203125,
           "text": "<image>"
         },
         {
@@ -4287,12 +4287,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.8359375,
+          "logprob": -13.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0234375,
+          "logprob": -12.0859375,
           "text": "<image>"
         },
         {
@@ -4302,12 +4302,7 @@
         },
         {
           "id": 32000,
-          "logprob": -14.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0546875,
+          "logprob": -14.0703125,
           "text": "<image>"
         },
         {
@@ -4317,7 +4312,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.0625,
+          "logprob": -14.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0390625,
           "text": "<image>"
         },
         {
@@ -4327,42 +4327,42 @@
         },
         {
           "id": 32000,
-          "logprob": -14.46875,
+          "logprob": -14.453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.703125,
+          "logprob": -14.6875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.296875,
+          "logprob": -14.3046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.8828125,
+          "logprob": -14.8671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1796875,
+          "logprob": -14.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2265625,
+          "logprob": -13.203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1171875,
+          "logprob": -13.109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9375,
+          "logprob": -11.8515625,
           "text": "<image>"
         },
         {
@@ -4377,12 +4377,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.71875,
+          "logprob": -12.7109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.734375,
+          "logprob": -13.7109375,
           "text": "<image>"
         },
         {
@@ -4392,32 +4392,32 @@
         },
         {
           "id": 32000,
-          "logprob": -12.46875,
+          "logprob": -12.4765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.15625,
+          "logprob": -12.140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.375,
+          "logprob": -12.3828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8984375,
+          "logprob": -12.875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.6953125,
+          "logprob": -13.6484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0625,
+          "logprob": -13.0390625,
           "text": "<image>"
         },
         {
@@ -4427,12 +4427,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.6796875,
+          "logprob": -12.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3515625,
+          "logprob": -12.34375,
           "text": "<image>"
         },
         {
@@ -4442,37 +4442,37 @@
         },
         {
           "id": 32000,
-          "logprob": -12.46875,
+          "logprob": -12.453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.03125,
+          "logprob": -12.015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.96875,
+          "logprob": -11.9765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.4609375,
+          "logprob": -14.46875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.328125,
+          "logprob": -13.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6484375,
+          "logprob": -12.71875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.65625,
+          "logprob": -10.6484375,
           "text": "<image>"
         },
         {
@@ -4492,22 +4492,22 @@
         },
         {
           "id": 32000,
-          "logprob": -12.6328125,
+          "logprob": -12.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.265625,
+          "logprob": -11.2578125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.734375,
+          "logprob": -11.7265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.84375,
+          "logprob": -10.8515625,
           "text": "<image>"
         },
         {
@@ -4517,32 +4517,32 @@
         },
         {
           "id": 32000,
-          "logprob": -13.109375,
+          "logprob": -13.140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7421875,
+          "logprob": -12.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.09375,
+          "logprob": -12.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.109375,
+          "logprob": -13.1328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.671875,
+          "logprob": -12.65625,
           "text": "<image>"
         },
         {
@@ -4552,27 +4552,27 @@
         },
         {
           "id": 32000,
-          "logprob": -12.6796875,
+          "logprob": -12.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.109375,
+          "logprob": -12.0859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.625,
+          "logprob": -13.609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8515625,
+          "logprob": -12.8359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3046875,
+          "logprob": -12.3125,
           "text": "<image>"
         },
         {
@@ -4582,12 +4582,12 @@
         },
         {
           "id": 32000,
-          "logprob": -14.0703125,
+          "logprob": -14.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.0,
+          "logprob": -13.984375,
           "text": "<image>"
         },
         {
@@ -4597,7 +4597,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.2421875,
+          "logprob": -13.2109375,
           "text": "<image>"
         },
         {
@@ -4607,42 +4607,42 @@
         },
         {
           "id": 32000,
-          "logprob": -13.7109375,
+          "logprob": -13.703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6015625,
+          "logprob": -12.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.796875,
+          "logprob": -15.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.90625,
+          "logprob": -11.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.375,
+          "logprob": -12.3671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4765625,
+          "logprob": -11.46875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.015625,
+          "logprob": -12.0,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.328125,
+          "logprob": -11.3359375,
           "text": "<image>"
         },
         {
@@ -4657,17 +4657,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.8203125,
+          "logprob": -13.828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7578125,
+          "logprob": -11.75,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.484375,
+          "logprob": -11.46875,
           "text": "<image>"
         },
         {
@@ -4682,17 +4682,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.640625,
+          "logprob": -11.625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5859375,
+          "logprob": -11.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.2578125,
+          "logprob": -15.2265625,
           "text": "<image>"
         },
         {
@@ -4702,17 +4702,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.0078125,
+          "logprob": -13.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.3828125,
+          "logprob": -15.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.203125,
+          "logprob": -11.1875,
           "text": "<image>"
         },
         {
@@ -4722,7 +4722,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.0703125,
+          "logprob": -11.0625,
           "text": "<image>"
         },
         {
@@ -4732,7 +4732,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.4453125,
+          "logprob": -13.453125,
           "text": "<image>"
         },
         {
@@ -4742,27 +4742,27 @@
         },
         {
           "id": 32000,
-          "logprob": -11.5078125,
+          "logprob": -11.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.015625,
+          "logprob": -12.0078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.15625,
+          "logprob": -11.1875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8359375,
+          "logprob": -11.890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.359375,
+          "logprob": -11.3125,
           "text": "<image>"
         },
         {
@@ -4777,22 +4777,22 @@
         },
         {
           "id": 32000,
-          "logprob": -12.4453125,
+          "logprob": -12.4921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.015625,
+          "logprob": -12.0703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.375,
+          "logprob": -13.3671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2734375,
+          "logprob": -13.28125,
           "text": "<image>"
         },
         {
@@ -4802,22 +4802,22 @@
         },
         {
           "id": 32000,
-          "logprob": -13.046875,
+          "logprob": -13.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0859375,
+          "logprob": -13.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1640625,
+          "logprob": -12.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4765625,
+          "logprob": -13.484375,
           "text": "<image>"
         },
         {
@@ -4827,7 +4827,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.7265625,
+          "logprob": -13.703125,
           "text": "<image>"
         },
         {
@@ -4837,262 +4837,17 @@
         },
         {
           "id": 32000,
-          "logprob": -14.3359375,
+          "logprob": -14.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.71875,
+          "logprob": -12.7109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.640625,
+          "logprob": -14.28125,
           "text": "<image>"
         },
         {
@@ -5102,52 +4857,57 @@
         },
         {
           "id": 32000,
-          "logprob": -13.453125,
+          "logprob": -13.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.4375,
+          "logprob": -15.203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1171875,
+          "logprob": -13.28125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4296875,
+          "logprob": -12.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.515625,
+          "logprob": -14.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.21875,
+          "logprob": -13.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8671875,
+          "logprob": -12.1171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8515625,
+          "logprob": -14.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.8671875,
+          "logprob": -14.0625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.59375,
+          "logprob": -12.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.953125,
           "text": "<image>"
         },
         {
@@ -5157,12 +4917,57 @@
         },
         {
           "id": 32000,
-          "logprob": -12.1640625,
+          "logprob": -11.6953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2265625,
+          "logprob": -12.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5234375,
           "text": "<image>"
         },
         {
@@ -5172,32 +4977,27 @@
         },
         {
           "id": 32000,
-          "logprob": -12.1015625,
+          "logprob": -11.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0234375,
+          "logprob": -11.2578125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4140625,
+          "logprob": -13.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.828125,
+          "logprob": -11.921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1953125,
+          "logprob": -13.03125,
           "text": "<image>"
         },
         {
@@ -5207,7 +5007,207 @@
         },
         {
           "id": 32000,
-          "logprob": -11.1328125,
+          "logprob": -14.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.4296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.140625,
           "text": "<image>"
         },
         {
@@ -5232,7 +5232,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.1015625,
+          "logprob": -11.1171875,
           "text": "<image>"
         },
         {
@@ -5242,7 +5242,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.15625,
+          "logprob": -11.1484375,
           "text": "<image>"
         },
         {
@@ -5252,17 +5252,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.4296875,
+          "logprob": -11.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.765625,
+          "logprob": -12.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.09375,
+          "logprob": -13.0859375,
           "text": "<image>"
         },
         {
@@ -5272,22 +5272,22 @@
         },
         {
           "id": 32000,
-          "logprob": -12.96875,
+          "logprob": -12.9609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.328125,
+          "logprob": -12.34375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8359375,
+          "logprob": -12.84375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.9609375,
+          "logprob": -13.9765625,
           "text": "<image>"
         },
         {
@@ -5297,7 +5297,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.46875,
+          "logprob": -13.484375,
           "text": "<image>"
         },
         {
@@ -5307,22 +5307,22 @@
         },
         {
           "id": 32000,
-          "logprob": -13.59375,
+          "logprob": -13.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.40625,
+          "logprob": -13.3828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.234375,
+          "logprob": -14.25,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.265625,
+          "logprob": -14.2578125,
           "text": "<image>"
         },
         {
@@ -5332,127 +5332,127 @@
         },
         {
           "id": 32000,
-          "logprob": -13.65625,
+          "logprob": -13.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.2734375,
+          "logprob": -14.2578125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.09375,
+          "logprob": -13.8671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.890625,
+          "logprob": -12.921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.640625,
+          "logprob": -15.6484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0234375,
+          "logprob": -13.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3828125,
+          "logprob": -12.34375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.71875,
+          "logprob": -12.7265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.5234375,
+          "logprob": -14.515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7734375,
+          "logprob": -12.75,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3203125,
+          "logprob": -12.265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.609375,
+          "logprob": -13.546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1640625,
+          "logprob": -13.1328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.203125,
+          "logprob": -12.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1875,
+          "logprob": -12.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.59375,
+          "logprob": -11.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6875,
+          "logprob": -11.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1796875,
+          "logprob": -12.15625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6484375,
+          "logprob": -11.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.421875,
+          "logprob": -11.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.34375,
+          "logprob": -11.390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1015625,
+          "logprob": -11.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9765625,
+          "logprob": -11.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.015625,
+          "logprob": -12.0078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0,
+          "logprob": -12.984375,
           "text": "<image>"
         },
         {
@@ -5462,7 +5462,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.09375,
           "text": "<image>"
         },
         {
@@ -5482,12 +5482,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.65625,
+          "logprob": -11.6484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.84375,
+          "logprob": -12.8125,
           "text": "<image>"
         },
         {
@@ -5497,17 +5497,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.6953125,
+          "logprob": -11.6796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.578125,
+          "logprob": -11.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.6796875,
+          "logprob": -13.6953125,
           "text": "<image>"
         },
         {
@@ -5522,7 +5522,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0078125,
+          "logprob": -12.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.296875,
           "text": "<image>"
         },
         {
@@ -5532,17 +5537,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.234375,
+          "logprob": -13.84375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8671875,
+          "logprob": -12.875,
           "text": "<image>"
         },
         {
@@ -5552,17 +5552,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.5078125,
+          "logprob": -13.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.953125,
+          "logprob": -13.9609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5078125,
+          "logprob": -12.46875,
           "text": "<image>"
         },
         {
@@ -5572,232 +5572,17 @@
         },
         {
           "id": 32000,
-          "logprob": -14.25,
+          "logprob": -14.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9140625,
+          "logprob": -12.921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4375,
+          "logprob": -14.2578125,
           "text": "<image>"
         },
         {
@@ -5807,62 +5592,62 @@
         },
         {
           "id": 32000,
-          "logprob": -13.78125,
+          "logprob": -14.3984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5703125,
+          "logprob": -15.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9765625,
+          "logprob": -13.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4765625,
+          "logprob": -12.3515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.125,
+          "logprob": -13.46875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0859375,
+          "logprob": -11.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9296875,
+          "logprob": -12.2890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.90625,
+          "logprob": -14.3515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0625,
+          "logprob": -13.140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0234375,
+          "logprob": -13.0234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.6953125,
+          "logprob": -12.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.890625,
+          "logprob": -11.8125,
           "text": "<image>"
         },
         {
@@ -5870,6 +5655,31 @@
           "logprob": -12.125,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -11.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.640625,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -12.4375,
@@ -5877,67 +5687,257 @@
         },
         {
           "id": 32000,
-          "logprob": -12.375,
+          "logprob": -11.46875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.28125,
+          "logprob": -11.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4921875,
+          "logprob": -12.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1640625,
+          "logprob": -11.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1953125,
+          "logprob": -11.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.4140625,
+          "logprob": -12.4453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6171875,
+          "logprob": -11.34375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.921875,
+          "logprob": -12.1015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6328125,
+          "logprob": -11.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2421875,
+          "logprob": -14.6015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0234375,
+          "logprob": -12.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6953125,
+          "logprob": -11.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1484375,
+          "logprob": -11.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1328125,
           "text": "<image>"
         },
         {
@@ -5947,27 +5947,27 @@
         },
         {
           "id": 32000,
-          "logprob": -11.7265625,
+          "logprob": -11.71875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6796875,
+          "logprob": -11.6875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4921875,
+          "logprob": -12.484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.875,
+          "logprob": -11.8828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5390625,
+          "logprob": -12.53125,
           "text": "<image>"
         },
         {
@@ -5982,27 +5982,27 @@
         },
         {
           "id": 32000,
-          "logprob": -13.71875,
+          "logprob": -13.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.78125,
+          "logprob": -14.8046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.578125,
+          "logprob": -11.609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5859375,
+          "logprob": -13.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.390625,
+          "logprob": -12.421875,
           "text": "<image>"
         },
         {
@@ -6012,17 +6012,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.65625,
+          "logprob": -13.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5,
+          "logprob": -12.5078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.109375,
+          "logprob": -13.125,
           "text": "<image>"
         },
         {
@@ -6032,37 +6032,37 @@
         },
         {
           "id": 32000,
-          "logprob": -12.2578125,
+          "logprob": -12.2890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7421875,
+          "logprob": -12.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1640625,
+          "logprob": -13.15625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9375,
+          "logprob": -11.9453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.265625,
+          "logprob": -12.28125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1640625,
+          "logprob": -12.1328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.8671875,
+          "logprob": -13.8203125,
           "text": "<image>"
         },
         {
@@ -6072,57 +6072,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.046875,
+          "logprob": -13.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.984375,
+          "logprob": -15.6953125,
           "text": "<image>"
         },
         {
@@ -6132,12 +6087,47 @@
         },
         {
           "id": 32000,
-          "logprob": -11.3984375,
+          "logprob": -14.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.796875,
+          "logprob": -13.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.234375,
           "text": "<image>"
         },
         {
@@ -6147,37 +6137,47 @@
         },
         {
           "id": 32000,
-          "logprob": -11.4296875,
+          "logprob": -11.828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9921875,
+          "logprob": -11.453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.59375,
+          "logprob": -11.453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8359375,
+          "logprob": -11.984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7421875,
+          "logprob": -11.609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.125,
+          "logprob": -11.8046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.875,
+          "logprob": -11.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8515625,
           "text": "<image>"
         },
         {
@@ -6187,27 +6187,27 @@
         },
         {
           "id": 32000,
-          "logprob": -10.796875,
+          "logprob": -10.8046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4765625,
+          "logprob": -11.578125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1484375,
+          "logprob": -11.15625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8046875,
+          "logprob": -12.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3828125,
+          "logprob": -11.3671875,
           "text": "<image>"
         },
         {
@@ -6227,7 +6227,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.5546875,
+          "logprob": -12.5625,
           "text": "<image>"
         },
         {
@@ -6237,367 +6237,57 @@
         },
         {
           "id": 32000,
-          "logprob": -10.40625,
+          "logprob": -10.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.59375,
+          "logprob": -12.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.03125,
+          "logprob": -12.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.03125,
+          "logprob": -12.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.65625,
+          "logprob": -12.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6328125,
+          "logprob": -11.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5,
+          "logprob": -13.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.078125,
+          "logprob": -12.0703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0234375,
+          "logprob": -13.015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.171875,
+          "logprob": -14.1484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.4609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.015625,
+          "logprob": -12.3671875,
           "text": "<image>"
         },
         {
@@ -6607,42 +6297,112 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0859375,
+          "logprob": -12.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.140625,
+          "logprob": -13.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.65625,
+          "logprob": -13.953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.890625,
+          "logprob": -12.4765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.59375,
+          "logprob": -14.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4375,
+          "logprob": -12.6015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4375,
+          "logprob": -13.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.109375,
+          "logprob": -15.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0546875,
           "text": "<image>"
         },
         {
@@ -6652,17 +6412,257 @@
         },
         {
           "id": 32000,
-          "logprob": -11.1015625,
+          "logprob": -11.9609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.203125,
+          "logprob": -14.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9921875,
+          "logprob": -12.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.1484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.96875,
           "text": "<image>"
         },
         {
@@ -6672,32 +6672,32 @@
         },
         {
           "id": 32000,
-          "logprob": -11.40625,
+          "logprob": -11.4296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.53125,
+          "logprob": -12.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.109375,
+          "logprob": -12.1015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3203125,
+          "logprob": -11.3125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.125,
+          "logprob": -11.1171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7578125,
+          "logprob": -11.75,
           "text": "<image>"
         },
         {
@@ -6712,7 +6712,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.109375,
+          "logprob": -11.1015625,
           "text": "<image>"
         },
         {
@@ -6732,67 +6732,67 @@
         },
         {
           "id": 32000,
-          "logprob": -12.609375,
+          "logprob": -12.6015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.65625,
+          "logprob": -12.6796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7890625,
+          "logprob": -12.8359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.28125,
+          "logprob": -12.296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4140625,
+          "logprob": -13.3671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.96875,
+          "logprob": -12.9765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1171875,
+          "logprob": -12.109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5390625,
+          "logprob": -12.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.609375,
+          "logprob": -12.6015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6484375,
+          "logprob": -12.625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.09375,
+          "logprob": -12.109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9921875,
+          "logprob": -13.0,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4453125,
+          "logprob": -13.4296875,
           "text": "<image>"
         },
         {
@@ -6802,17 +6802,17 @@
         },
         {
           "id": 32000,
-          "logprob": -14.7109375,
+          "logprob": -14.71875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.625,
+          "logprob": -13.6015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.765625,
+          "logprob": -13.7578125,
           "text": "<image>"
         },
         {
@@ -6822,37 +6822,37 @@
         },
         {
           "id": 32000,
-          "logprob": -14.53125,
+          "logprob": -14.5390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.34375,
+          "logprob": -13.3515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.921875,
+          "logprob": -12.9453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.359375,
+          "logprob": -11.3515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.078125,
+          "logprob": -12.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4609375,
+          "logprob": -11.421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.640625,
+          "logprob": -11.625,
           "text": "<image>"
         },
         {
@@ -6867,62 +6867,62 @@
         },
         {
           "id": 32000,
-          "logprob": -11.953125,
+          "logprob": -11.9453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0859375,
+          "logprob": -12.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.421875,
+          "logprob": -11.4296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5859375,
+          "logprob": -11.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0859375,
+          "logprob": -12.0703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.25,
+          "logprob": -11.2421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6171875,
+          "logprob": -11.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3359375,
+          "logprob": -11.3125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.46875,
+          "logprob": -11.453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.265625,
+          "logprob": -12.28125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.21875,
+          "logprob": -12.2109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1875,
+          "logprob": -12.203125,
           "text": "<image>"
         },
         {
@@ -6937,47 +6937,47 @@
         },
         {
           "id": 32000,
-          "logprob": -11.2421875,
+          "logprob": -11.25,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3046875,
+          "logprob": -12.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.03125,
+          "logprob": -11.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.203125,
+          "logprob": -11.1953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.125,
+          "logprob": -11.1171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9296875,
+          "logprob": -11.9375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9765625,
+          "logprob": -10.984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.453125,
+          "logprob": -11.390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.859375,
+          "logprob": -12.8828125,
           "text": "<image>"
         },
         {
@@ -7002,47 +7002,47 @@
         },
         {
           "id": 32000,
-          "logprob": -12.1328125,
+          "logprob": -12.1171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5,
+          "logprob": -12.515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8671875,
+          "logprob": -11.890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1640625,
+          "logprob": -13.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.765625,
+          "logprob": -15.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.609375,
+          "logprob": -11.6015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.34375,
+          "logprob": -13.328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.671875,
+          "logprob": -13.6484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.9140625,
+          "logprob": -14.875,
           "text": "<image>"
         },
         {
@@ -7052,57 +7052,87 @@
         },
         {
           "id": 32000,
-          "logprob": -13.53125,
+          "logprob": -13.5234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.25,
+          "logprob": -13.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6171875,
+          "logprob": -12.609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.8046875,
+          "logprob": -13.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.375,
+          "logprob": -14.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.171875,
+          "logprob": -12.1484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.828125,
+          "logprob": -11.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.875,
+          "logprob": -11.890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.484375,
+          "logprob": -11.4921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5625,
+          "logprob": -12.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.640625,
+          "logprob": -11.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.03125,
           "text": "<image>"
         },
         {
@@ -7110,36 +7140,6 @@
           "logprob": -11.7265625,
           "text": "<image>"
         },
-        {
-          "id": 32000,
-          "logprob": -11.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.734375,
-          "text": "<image>"
-        },
         {
           "id": 32000,
           "logprob": -11.84375,
@@ -7147,12 +7147,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0390625,
+          "logprob": -11.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.2578125,
+          "logprob": -11.2421875,
           "text": "<image>"
         },
         {
@@ -7187,22 +7187,22 @@
         },
         {
           "id": 32000,
-          "logprob": -11.0390625,
+          "logprob": -11.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1953125,
+          "logprob": -12.203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.6640625,
+          "logprob": -13.609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6640625,
+          "logprob": -12.65625,
           "text": "<image>"
         },
         {
@@ -7227,27 +7227,47 @@
         },
         {
           "id": 32000,
-          "logprob": -11.9140625,
+          "logprob": -11.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.34375,
+          "logprob": -11.3359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.578125,
+          "logprob": -11.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.953125,
+          "logprob": -11.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.296875,
+          "logprob": -11.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.875,
           "text": "<image>"
         },
         {
@@ -7257,27 +7277,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9765625,
+          "logprob": -12.046875,
           "text": "<image>"
         },
         {
@@ -7287,7 +7287,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.734375,
+          "logprob": -13.7265625,
           "text": "<image>"
         },
         {
@@ -7297,7 +7297,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.8359375,
+          "logprob": -13.8203125,
           "text": "<image>"
         },
         {
@@ -7312,37 +7312,37 @@
         },
         {
           "id": 32000,
-          "logprob": -14.5625,
+          "logprob": -14.546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.0234375,
+          "logprob": -14.015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5703125,
+          "logprob": -12.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8125,
+          "logprob": -11.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.7109375,
+          "logprob": -10.6953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6015625,
+          "logprob": -11.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8671875,
+          "logprob": -12.0234375,
           "text": "<image>"
         },
         {
@@ -7357,32 +7357,32 @@
         },
         {
           "id": 32000,
-          "logprob": -14.3359375,
+          "logprob": -14.34375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.90625,
+          "logprob": -11.8984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.296875,
+          "logprob": -13.4375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.984375,
+          "logprob": -11.9765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0625,
+          "logprob": -11.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4296875,
+          "logprob": -11.421875,
           "text": "<image>"
         },
         {
@@ -7397,27 +7397,27 @@
         },
         {
           "id": 32000,
-          "logprob": -11.28125,
+          "logprob": -11.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5546875,
+          "logprob": -11.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.671875,
+          "logprob": -11.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1171875,
+          "logprob": -12.125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5,
+          "logprob": -11.484375,
           "text": "<image>"
         },
         {
@@ -7427,27 +7427,27 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8984375,
+          "logprob": -11.875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2890625,
+          "logprob": -12.3125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6484375,
+          "logprob": -11.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8984375,
+          "logprob": -12.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.734375,
+          "logprob": -11.7421875,
           "text": "<image>"
         },
         {
@@ -7457,7 +7457,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.40625,
+          "logprob": -11.3984375,
           "text": "<image>"
         },
         {
@@ -7467,7 +7467,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.3515625,
+          "logprob": -11.34375,
           "text": "<image>"
         },
         {
@@ -7482,37 +7482,37 @@
         },
         {
           "id": 32000,
-          "logprob": -11.03125,
+          "logprob": -10.9609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.375,
+          "logprob": -13.265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6640625,
+          "logprob": -12.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.96875,
+          "logprob": -11.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.671875,
+          "logprob": -14.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.390625,
+          "logprob": -12.3828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.2890625,
+          "logprob": -14.3515625,
           "text": "<image>"
         },
         {
@@ -7522,32 +7522,32 @@
         },
         {
           "id": 32000,
-          "logprob": -15.1796875,
+          "logprob": -15.125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6875,
+          "logprob": -12.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7421875,
+          "logprob": -12.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.6484375,
+          "logprob": -14.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.546875,
+          "logprob": -13.5234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5703125,
+          "logprob": -12.5859375,
           "text": "<image>"
         },
         {
@@ -7557,32 +7557,17 @@
         },
         {
           "id": 32000,
-          "logprob": -14.390625,
+          "logprob": -14.3828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.96875,
+          "logprob": -12.9453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7421875,
+          "logprob": -12.609375,
           "text": "<image>"
         },
         {
@@ -7592,32 +7577,47 @@
         },
         {
           "id": 32000,
-          "logprob": -12.453125,
+          "logprob": -12.953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.8046875,
+          "logprob": -11.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.0078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.7890625,
+          "logprob": -12.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.3828125,
+          "logprob": -13.8828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.828125,
+          "logprob": -12.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8359375,
           "text": "<image>"
         },
         {
@@ -7627,12 +7627,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.0,
+          "logprob": -13.0078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1171875,
+          "logprob": -12.140625,
           "text": "<image>"
         },
         {
@@ -7642,17 +7642,17 @@
         },
         {
           "id": 32000,
-          "logprob": -14.015625,
+          "logprob": -13.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6328125,
+          "logprob": -11.625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4765625,
+          "logprob": -11.484375,
           "text": "<image>"
         },
         {
@@ -7662,7 +7662,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.5859375,
+          "logprob": -11.5703125,
           "text": "<image>"
         },
         {
@@ -7677,7 +7677,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0390625,
+          "logprob": -12.03125,
           "text": "<image>"
         },
         {
@@ -7687,7 +7687,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.4609375,
+          "logprob": -11.46875,
           "text": "<image>"
         },
         {
@@ -7697,7 +7697,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0625,
+          "logprob": -12.078125,
           "text": "<image>"
         },
         {
@@ -7707,142 +7707,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
+          "logprob": -11.296875,
           "text": "<image>"
         },
         {
@@ -7852,7 +7717,142 @@
         },
         {
           "id": 32000,
-          "logprob": -11.34375,
+          "logprob": -11.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.21875,
           "text": "<image>"
         },
         {
@@ -7862,52 +7862,52 @@
         },
         {
           "id": 32000,
-          "logprob": -11.3671875,
+          "logprob": -11.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0,
+          "logprob": -12.984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.1875,
+          "logprob": -15.15625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0,
+          "logprob": -11.9765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.75,
+          "logprob": -12.765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.515625,
+          "logprob": -13.4921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5703125,
+          "logprob": -11.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.703125,
+          "logprob": -11.6953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3671875,
+          "logprob": -11.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4921875,
+          "logprob": -11.5,
           "text": "<image>"
         },
         {
@@ -7917,12 +7917,12 @@
         },
         {
           "id": 32000,
-          "logprob": -14.5546875,
+          "logprob": -14.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.390625,
+          "logprob": -13.4140625,
           "text": "<image>"
         },
         {
@@ -7932,7 +7932,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.7265625,
+          "logprob": -11.71875,
           "text": "<image>"
         },
         {
@@ -7947,7 +7947,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.2421875,
+          "logprob": -13.2578125,
           "text": "<image>"
         },
         {
@@ -7967,27 +7967,27 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8203125,
+          "logprob": -11.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9296875,
+          "logprob": -11.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.796875,
+          "logprob": -11.765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1875,
+          "logprob": -11.1953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.359375,
+          "logprob": -13.3359375,
           "text": "<image>"
         },
         {
@@ -7997,7 +7997,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.6796875,
+          "logprob": -12.703125,
           "text": "<image>"
         },
         {
@@ -8012,27 +8012,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.5625,
+          "logprob": -13.6171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6484375,
+          "logprob": -13.328125,
           "text": "<image>"
         },
         {
@@ -8040,6 +8025,21 @@
           "logprob": -12.828125,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -13.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.796875,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -12.78125,
@@ -8047,42 +8047,42 @@
         },
         {
           "id": 32000,
-          "logprob": -12.578125,
+          "logprob": -12.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7578125,
+          "logprob": -12.765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.484375,
+          "logprob": -11.46875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5390625,
+          "logprob": -12.484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6015625,
+          "logprob": -11.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.640625,
+          "logprob": -11.6171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.140625,
+          "logprob": -12.1328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.65625,
+          "logprob": -12.59375,
           "text": "<image>"
         },
         {
@@ -8092,17 +8092,17 @@
         },
         {
           "id": 32000,
-          "logprob": -14.265625,
+          "logprob": -14.25,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0078125,
+          "logprob": -12.0234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7578125,
+          "logprob": -11.7734375,
           "text": "<image>"
         },
         {
@@ -8117,52 +8117,52 @@
         },
         {
           "id": 32000,
-          "logprob": -11.625,
+          "logprob": -11.609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.53125,
+          "logprob": -13.5234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.046875,
+          "logprob": -12.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6796875,
+          "logprob": -12.6953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7109375,
+          "logprob": -11.703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.890625,
+          "logprob": -11.8828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8359375,
+          "logprob": -11.84375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7734375,
+          "logprob": -11.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3671875,
+          "logprob": -12.3515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.3125,
+          "logprob": -14.3046875,
           "text": "<image>"
         },
         {
@@ -8172,22 +8172,22 @@
         },
         {
           "id": 32000,
-          "logprob": -11.921875,
+          "logprob": -11.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3203125,
+          "logprob": -11.328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1640625,
+          "logprob": -14.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.2421875,
+          "logprob": -11.25,
           "text": "<image>"
         },
         {
@@ -8197,7 +8197,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.078125,
           "text": "<image>"
         },
         {
@@ -8207,12 +8207,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.2578125,
+          "logprob": -12.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.7265625,
+          "logprob": -13.71875,
           "text": "<image>"
         },
         {
@@ -8232,12 +8232,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.3828125,
+          "logprob": -13.390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5390625,
+          "logprob": -12.515625,
           "text": "<image>"
         },
         {
@@ -8247,22 +8247,22 @@
         },
         {
           "id": 32000,
-          "logprob": -14.15625,
+          "logprob": -14.1796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5703125,
+          "logprob": -11.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5546875,
+          "logprob": -11.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.65625,
+          "logprob": -11.671875,
           "text": "<image>"
         },
         {
@@ -8272,47 +8272,47 @@
         },
         {
           "id": 32000,
-          "logprob": -12.6484375,
+          "logprob": -12.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5390625,
+          "logprob": -12.515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7734375,
+          "logprob": -11.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0859375,
+          "logprob": -13.1328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9140625,
+          "logprob": -12.8984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.6015625,
+          "logprob": -14.609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5390625,
+          "logprob": -11.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8828125,
+          "logprob": -11.8515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.078125,
+          "logprob": -13.0625,
           "text": "<image>"
         },
         {
@@ -8322,17 +8322,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.15625,
+          "logprob": -12.1328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.03125,
+          "logprob": -12.0234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4453125,
+          "logprob": -12.453125,
           "text": "<image>"
         },
         {
@@ -8342,7 +8342,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.9375,
+          "logprob": -11.9296875,
           "text": "<image>"
         },
         {
@@ -8357,42 +8357,42 @@
         },
         {
           "id": 32000,
-          "logprob": -12.7890625,
+          "logprob": -12.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0390625,
+          "logprob": -13.0234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8203125,
+          "logprob": -12.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.578125,
+          "logprob": -12.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.7578125,
+          "logprob": -10.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9609375,
+          "logprob": -11.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.484375,
+          "logprob": -11.4921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.265625,
+          "logprob": -11.25,
           "text": "<image>"
         },
         {
@@ -8402,17 +8402,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.71875,
+          "logprob": -12.6953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7890625,
+          "logprob": -11.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5234375,
+          "logprob": -11.515625,
           "text": "<image>"
         },
         {
@@ -8422,7 +8422,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.46875,
+          "logprob": -12.4765625,
           "text": "<image>"
         },
         {
@@ -8437,17 +8437,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.65625,
+          "logprob": -11.6484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5234375,
+          "logprob": -11.515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7890625,
+          "logprob": -12.8125,
           "text": "<image>"
         },
         {
@@ -8457,17 +8457,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.28125,
+          "logprob": -13.2890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.109375,
+          "logprob": -11.09375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.34375,
+          "logprob": -12.3359375,
           "text": "<image>"
         },
         {
@@ -8477,27 +8477,27 @@
         },
         {
           "id": 32000,
-          "logprob": -13.0,
+          "logprob": -13.0078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.28125,
+          "logprob": -15.328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.8046875,
+          "logprob": -14.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0234375,
+          "logprob": -11.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.109375,
+          "logprob": -13.0703125,
           "text": "<image>"
         },
         {
@@ -8507,12 +8507,12 @@
         },
         {
           "id": 32000,
-          "logprob": -14.0078125,
+          "logprob": -14.0859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.7109375,
+          "logprob": -14.734375,
           "text": "<image>"
         },
         {
@@ -8527,127 +8527,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.9765625,
+          "logprob": -13.953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.3046875,
+          "logprob": -14.28125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.78125,
+          "logprob": -14.8828125,
           "text": "<image>"
         },
         {
@@ -8657,22 +8547,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.859375,
+          "logprob": -12.28125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.765625,
+          "logprob": -12.34375,
           "text": "<image>"
         },
         {
@@ -8687,117 +8567,87 @@
         },
         {
           "id": 32000,
-          "logprob": -11.71875,
+          "logprob": -12.4296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.25,
+          "logprob": -11.8984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.65625,
+          "logprob": -11.984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.84375,
+          "logprob": -11.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0390625,
+          "logprob": -11.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4921875,
+          "logprob": -12.1875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8828125,
+          "logprob": -11.6171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1015625,
+          "logprob": -12.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.0625,
+          "logprob": -12.4765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.09375,
+          "logprob": -12.953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2578125,
+          "logprob": -10.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.4453125,
+          "logprob": -13.1015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9921875,
+          "logprob": -12.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.140625,
+          "logprob": -11.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.828125,
+          "logprob": -11.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5546875,
+          "logprob": -11.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6015625,
+          "logprob": -11.765625,
           "text": "<image>"
         },
         {
@@ -8807,32 +8657,182 @@
         },
         {
           "id": 32000,
-          "logprob": -12.84375,
+          "logprob": -12.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.78125,
+          "logprob": -11.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1171875,
+          "logprob": -12.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8046875,
+          "logprob": -11.71875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5234375,
+          "logprob": -11.34375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4609375,
+          "logprob": -12.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4765625,
           "text": "<image>"
         },
         {
@@ -8852,22 +8852,22 @@
         },
         {
           "id": 32000,
-          "logprob": -12.953125,
+          "logprob": -12.9609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.859375,
+          "logprob": -12.8515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0390625,
+          "logprob": -12.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.28125,
+          "logprob": -13.234375,
           "text": "<image>"
         },
         {
@@ -8877,12 +8877,12 @@
         },
         {
           "id": 32000,
-          "logprob": -10.8203125,
+          "logprob": -10.8515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.6953125,
+          "logprob": -10.671875,
           "text": "<image>"
         },
         {
@@ -8892,7 +8892,7 @@
         },
         {
           "id": 32000,
-          "logprob": -10.9375,
+          "logprob": -10.9140625,
           "text": "<image>"
         },
         {
@@ -8912,17 +8912,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.3046875,
+          "logprob": -12.2890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9609375,
+          "logprob": -10.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6328125,
+          "logprob": -11.6015625,
           "text": "<image>"
         },
         {
@@ -8932,17 +8932,17 @@
         },
         {
           "id": 32000,
-          "logprob": -10.84375,
+          "logprob": -10.8515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.140625,
+          "logprob": -11.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.09375,
+          "logprob": -11.0859375,
           "text": "<image>"
         },
         {
@@ -8952,12 +8952,12 @@
         },
         {
           "id": 32000,
-          "logprob": -15.578125,
+          "logprob": -15.765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.875,
+          "logprob": -11.8515625,
           "text": "<image>"
         },
         {
@@ -8972,22 +8972,22 @@
         },
         {
           "id": 32000,
-          "logprob": -10.984375,
+          "logprob": -10.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.234375,
+          "logprob": -12.25,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8359375,
+          "logprob": -11.828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8515625,
+          "logprob": -11.84375,
           "text": "<image>"
         },
         {
@@ -9002,7 +9002,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.5625,
+          "logprob": -13.5703125,
           "text": "<image>"
         },
         {
@@ -9012,37 +9012,37 @@
         },
         {
           "id": 32000,
-          "logprob": -13.5859375,
+          "logprob": -13.6171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.0234375,
+          "logprob": -14.0078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1796875,
+          "logprob": -11.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1484375,
+          "logprob": -14.140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.234375,
+          "logprob": -12.25,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.734375,
+          "logprob": -11.7265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3515625,
+          "logprob": -12.3203125,
           "text": "<image>"
         },
         {
@@ -9052,17 +9052,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.90625,
+          "logprob": -11.8828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3359375,
+          "logprob": -12.3125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5703125,
+          "logprob": -11.5625,
           "text": "<image>"
         },
         {
@@ -9072,17 +9072,17 @@
         },
         {
           "id": 32000,
-          "logprob": -14.6875,
+          "logprob": -14.6796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0625,
+          "logprob": -11.046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5234375,
+          "logprob": -11.5,
           "text": "<image>"
         },
         {
@@ -9107,12 +9107,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.46875,
+          "logprob": -12.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.0859375,
           "text": "<image>"
         },
         {
@@ -9125,21 +9125,6 @@
           "logprob": -12.46875,
           "text": "<image>"
         },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6640625,
-          "text": "<image>"
-        },
         {
           "id": 32000,
           "logprob": -11.9765625,
@@ -9147,7 +9132,22 @@
         },
         {
           "id": 32000,
-          "logprob": -10.8671875,
+          "logprob": -11.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.875,
           "text": "<image>"
         },
         {
@@ -9167,12 +9167,12 @@
         },
         {
           "id": 32000,
-          "logprob": -10.796875,
+          "logprob": -10.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.59375,
+          "logprob": -11.6171875,
           "text": "<image>"
         },
         {
@@ -9202,22 +9202,22 @@
         },
         {
           "id": 32000,
-          "logprob": -11.390625,
+          "logprob": -11.3828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9375,
+          "logprob": -10.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.6796875,
+          "logprob": -10.703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9375,
+          "logprob": -11.90625,
           "text": "<image>"
         },
         {
@@ -9227,82 +9227,57 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8125,
+          "logprob": -11.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3671875,
+          "logprob": -11.4453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -9.625,
+          "logprob": -9.6328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.8515625,
+          "logprob": -10.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9453125,
+          "logprob": -13.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8203125,
+          "logprob": -12.890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3359375,
+          "logprob": -13.4765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.015625,
+          "logprob": -14.109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8671875,
+          "logprob": -11.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4453125,
+          "logprob": -13.484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.75,
+          "logprob": -13.5625,
           "text": "<image>"
         },
         {
@@ -9310,6 +9285,31 @@
           "logprob": -12.859375,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -10.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.890625,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -13.0625,
@@ -9317,7 +9317,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.21875,
+          "logprob": -11.2265625,
           "text": "<image>"
         },
         {
@@ -9327,7 +9327,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.296875,
+          "logprob": -12.2890625,
           "text": "<image>"
         },
         {
@@ -9337,12 +9337,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.8671875,
+          "logprob": -12.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.6953125,
+          "logprob": -15.671875,
           "text": "<image>"
         },
         {
@@ -9357,7 +9357,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.140625,
+          "logprob": -12.1328125,
           "text": "<image>"
         },
         {
@@ -9367,22 +9367,22 @@
         },
         {
           "id": 32000,
-          "logprob": -11.390625,
+          "logprob": -11.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.328125,
+          "logprob": -12.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.265625,
+          "logprob": -12.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9453125,
+          "logprob": -10.9296875,
           "text": "<image>"
         },
         {
@@ -9397,7 +9397,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.6015625,
+          "logprob": -11.59375,
           "text": "<image>"
         },
         {
@@ -9407,7 +9407,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.546875,
+          "logprob": -11.53125,
           "text": "<image>"
         },
         {
@@ -9422,7 +9422,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.6796875,
+          "logprob": -11.6875,
           "text": "<image>"
         },
         {
@@ -9437,27 +9437,27 @@
         },
         {
           "id": 32000,
-          "logprob": -12.125,
+          "logprob": -12.1171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5703125,
+          "logprob": -12.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.109375,
+          "logprob": -12.1015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0078125,
+          "logprob": -12.0,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.734375,
+          "logprob": -13.7265625,
           "text": "<image>"
         },
         {
@@ -9467,67 +9467,67 @@
         },
         {
           "id": 32000,
-          "logprob": -11.6484375,
+          "logprob": -11.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.34375,
+          "logprob": -12.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.140625,
+          "logprob": -12.1484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1796875,
+          "logprob": -14.21875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.8046875,
+          "logprob": -14.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -18.046875,
+          "logprob": -17.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.34375,
+          "logprob": -15.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.4453125,
+          "logprob": -14.4765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.625,
+          "logprob": -10.6015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0546875,
+          "logprob": -12.0703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.859375,
+          "logprob": -12.84375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2265625,
+          "logprob": -12.2421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2109375,
+          "logprob": -13.203125,
           "text": "<image>"
         },
         {
@@ -9537,37 +9537,37 @@
         },
         {
           "id": 32000,
-          "logprob": -10.9296875,
+          "logprob": -10.9375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9140625,
+          "logprob": -11.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.125,
+          "logprob": -12.1015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9609375,
+          "logprob": -10.9453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7734375,
+          "logprob": -11.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.828125,
+          "logprob": -11.8203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2890625,
+          "logprob": -12.296875,
           "text": "<image>"
         },
         {
@@ -9577,27 +9577,27 @@
         },
         {
           "id": 32000,
-          "logprob": -11.53125,
+          "logprob": -11.5390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0234375,
+          "logprob": -12.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.2421875,
+          "logprob": -15.2265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1171875,
+          "logprob": -12.125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1171875,
+          "logprob": -12.125,
           "text": "<image>"
         },
         {
@@ -9607,7 +9607,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.671875,
+          "logprob": -11.6796875,
           "text": "<image>"
         },
         {
@@ -9627,32 +9627,32 @@
         },
         {
           "id": 32000,
-          "logprob": -12.734375,
+          "logprob": -12.703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.609375,
+          "logprob": -11.6328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5234375,
+          "logprob": -11.515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5078125,
+          "logprob": -11.546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2109375,
+          "logprob": -12.21875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.421875,
+          "logprob": -11.4140625,
           "text": "<image>"
         },
         {
@@ -9662,137 +9662,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.96875,
+          "logprob": -11.9765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.46875,
+          "logprob": -12.9609375,
           "text": "<image>"
         },
         {
@@ -9800,6 +9675,131 @@
           "logprob": -12.578125,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -11.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5078125,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -11.6171875,
@@ -9807,32 +9807,32 @@
         },
         {
           "id": 32000,
-          "logprob": -13.8125,
+          "logprob": -13.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1640625,
+          "logprob": -11.203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.765625,
+          "logprob": -11.78125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.5,
+          "logprob": -16.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3046875,
+          "logprob": -12.296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1171875,
+          "logprob": -14.1484375,
           "text": "<image>"
         },
         {
@@ -9842,12 +9842,47 @@
         },
         {
           "id": 32000,
-          "logprob": -12.5,
+          "logprob": -12.4921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.6015625,
+          "logprob": -10.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1953125,
           "text": "<image>"
         },
         {
@@ -9857,52 +9892,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.625,
+          "logprob": -11.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3046875,
+          "logprob": -12.6484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1171875,
+          "logprob": -11.109375,
           "text": "<image>"
         },
         {
@@ -9912,7 +9912,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.921875,
+          "logprob": -13.90625,
           "text": "<image>"
         },
         {
@@ -9927,12 +9927,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.3671875,
+          "logprob": -13.375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.7109375,
+          "logprob": -14.703125,
           "text": "<image>"
         },
         {
@@ -9942,7 +9942,7 @@
         },
         {
           "id": 32000,
-          "logprob": -14.859375,
+          "logprob": -14.890625,
           "text": "<image>"
         },
         {
@@ -9957,37 +9957,37 @@
         },
         {
           "id": 32000,
-          "logprob": -11.9296875,
+          "logprob": -11.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3828125,
+          "logprob": -13.3984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.046875,
+          "logprob": -13.9453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1484375,
+          "logprob": -13.1875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.265625,
+          "logprob": -11.28125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.125,
+          "logprob": -15.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.046875,
+          "logprob": -15.0234375,
           "text": "<image>"
         },
         {
@@ -10002,92 +10002,22 @@
         },
         {
           "id": 32000,
-          "logprob": -14.09375,
+          "logprob": -14.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.890625,
+          "logprob": -13.8984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.3203125,
+          "logprob": -15.3125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9609375,
+          "logprob": -12.8125,
           "text": "<image>"
         },
         {
@@ -10097,22 +10027,92 @@
         },
         {
           "id": 32000,
-          "logprob": -12.3203125,
+          "logprob": -13.046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.59375,
+          "logprob": -10.625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.171875,
+          "logprob": -11.25,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5,
+          "logprob": -12.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4921875,
           "text": "<image>"
         },
         {
@@ -10142,17 +10142,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.78125,
+          "logprob": -12.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3125,
+          "logprob": -11.3046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1171875,
+          "logprob": -11.109375,
           "text": "<image>"
         },
         {
@@ -10162,27 +10162,27 @@
         },
         {
           "id": 32000,
-          "logprob": -13.0859375,
+          "logprob": -13.0234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.921875,
+          "logprob": -10.9296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.515625,
+          "logprob": -13.5078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4921875,
+          "logprob": -11.5,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8203125,
+          "logprob": -11.8359375,
           "text": "<image>"
         },
         {
@@ -10192,42 +10192,42 @@
         },
         {
           "id": 32000,
-          "logprob": -14.0078125,
+          "logprob": -14.0234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.21875,
+          "logprob": -15.2578125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -17.84375,
+          "logprob": -17.953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.046875,
+          "logprob": -12.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.296875,
+          "logprob": -12.28125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7109375,
+          "logprob": -11.6875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1015625,
+          "logprob": -12.1171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.515625,
+          "logprob": -11.5546875,
           "text": "<image>"
         },
         {
@@ -10237,17 +10237,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.640625,
+          "logprob": -12.6171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4609375,
+          "logprob": -13.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.84375,
+          "logprob": -14.828125,
           "text": "<image>"
         },
         {
@@ -10257,22 +10257,22 @@
         },
         {
           "id": 32000,
-          "logprob": -11.6171875,
+          "logprob": -11.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.765625,
+          "logprob": -12.7421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5390625,
+          "logprob": -13.484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4453125,
+          "logprob": -11.4609375,
           "text": "<image>"
         },
         {
@@ -10282,37 +10282,37 @@
         },
         {
           "id": 32000,
-          "logprob": -11.96875,
+          "logprob": -11.984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4765625,
+          "logprob": -12.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.890625,
+          "logprob": -15.9375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.953125,
+          "logprob": -11.984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.40625,
+          "logprob": -11.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0,
+          "logprob": -12.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1875,
+          "logprob": -14.3046875,
           "text": "<image>"
         },
         {
@@ -10322,7 +10322,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.125,
+          "logprob": -12.1171875,
           "text": "<image>"
         },
         {
@@ -10332,22 +10332,22 @@
         },
         {
           "id": 32000,
-          "logprob": -13.140625,
+          "logprob": -13.1484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1484375,
+          "logprob": -12.1875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.375,
+          "logprob": -13.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.9453125,
+          "logprob": -13.9921875,
           "text": "<image>"
         },
         {
@@ -10357,12 +10357,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0859375,
+          "logprob": -12.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.734375,
+          "logprob": -10.7421875,
           "text": "<image>"
         },
         {
@@ -10372,12 +10372,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.65625,
+          "logprob": -12.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.953125,
+          "logprob": -11.9765625,
           "text": "<image>"
         },
         {
@@ -10387,12 +10387,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.5,
+          "logprob": -12.4765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8984375,
+          "logprob": -11.890625,
           "text": "<image>"
         },
         {
@@ -10412,22 +10412,22 @@
         },
         {
           "id": 32000,
-          "logprob": -12.4375,
+          "logprob": -12.4609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5703125,
+          "logprob": -12.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.828125,
+          "logprob": -12.8203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.375,
+          "logprob": -13.359375,
           "text": "<image>"
         },
         {
@@ -10447,7 +10447,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.2421875,
+          "logprob": -11.234375,
           "text": "<image>"
         },
         {
@@ -10457,27 +10457,27 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.328125,
+          "logprob": -15.3046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.890625,
+          "logprob": -12.8828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4453125,
+          "logprob": -13.453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8203125,
+          "logprob": -11.8125,
           "text": "<image>"
         },
         {
@@ -10487,17 +10487,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.96875,
+          "logprob": -12.9765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9921875,
+          "logprob": -11.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.421875,
+          "logprob": -13.4296875,
           "text": "<image>"
         },
         {
@@ -10507,12 +10507,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.265625,
+          "logprob": -13.28125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.265625,
+          "logprob": -13.2578125,
           "text": "<image>"
         },
         {
@@ -10527,12 +10527,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.15625,
+          "logprob": -12.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.375,
+          "logprob": -10.3828125,
           "text": "<image>"
         },
         {
@@ -10542,7 +10542,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.2109375,
+          "logprob": -13.203125,
           "text": "<image>"
         },
         {
@@ -10552,7 +10552,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.15625,
+          "logprob": -12.1328125,
           "text": "<image>"
         },
         {
@@ -10562,12 +10562,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.171875,
+          "logprob": -13.1796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6328125,
+          "logprob": -11.65625,
           "text": "<image>"
         },
         {
@@ -10577,12 +10577,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.90625,
+          "logprob": -11.9375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.765625,
+          "logprob": -11.7734375,
           "text": "<image>"
         },
         {
@@ -10597,7 +10597,7 @@
         },
         {
           "id": 32000,
-          "logprob": -10.8984375,
+          "logprob": -10.90625,
           "text": "<image>"
         },
         {
@@ -10607,17 +10607,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.40625,
+          "logprob": -12.3828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5390625,
+          "logprob": -11.5234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.59375,
+          "logprob": -16.578125,
           "text": "<image>"
         },
         {
@@ -10627,7 +10627,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.9140625,
+          "logprob": -11.9453125,
           "text": "<image>"
         },
         {
@@ -10642,22 +10642,22 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8984375,
+          "logprob": -11.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0703125,
+          "logprob": -13.0625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.03125,
+          "logprob": -12.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.921875,
+          "logprob": -11.9296875,
           "text": "<image>"
         },
         {
@@ -10667,12 +10667,12 @@
         },
         {
           "id": 32000,
-          "logprob": -10.5390625,
+          "logprob": -10.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.453125,
+          "logprob": -11.4375,
           "text": "<image>"
         },
         {
@@ -10682,22 +10682,22 @@
         },
         {
           "id": 32000,
-          "logprob": -13.0859375,
+          "logprob": -13.09375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1796875,
+          "logprob": -12.15625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.8203125,
+          "logprob": -15.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.984375,
+          "logprob": -10.9765625,
           "text": "<image>"
         },
         {
@@ -10707,42 +10707,42 @@
         },
         {
           "id": 32000,
-          "logprob": -10.109375,
+          "logprob": -10.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8671875,
+          "logprob": -11.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1328125,
+          "logprob": -13.125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.640625,
+          "logprob": -12.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.015625,
+          "logprob": -16.046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5078125,
+          "logprob": -11.515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.7265625,
+          "logprob": -13.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.703125,
+          "logprob": -14.671875,
           "text": "<image>"
         },
         {
@@ -10752,32 +10752,32 @@
         },
         {
           "id": 32000,
-          "logprob": -12.2421875,
+          "logprob": -12.2109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.6640625,
+          "logprob": -10.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.6640625,
+          "logprob": -15.6484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9140625,
+          "logprob": -10.9296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3203125,
+          "logprob": -11.328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.046875,
+          "logprob": -11.0546875,
           "text": "<image>"
         },
         {
@@ -10787,7 +10787,7 @@
         },
         {
           "id": 32000,
-          "logprob": -10.953125,
+          "logprob": -10.96875,
           "text": "<image>"
         },
         {
@@ -10802,337 +10802,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2578125,
+          "logprob": -12.359375,
           "text": "<image>"
         },
         {
@@ -11142,22 +10812,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.84375,
+          "logprob": -14.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
+          "logprob": -11.4921875,
           "text": "<image>"
         },
         {
@@ -11167,87 +10827,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.125,
+          "logprob": -12.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.390625,
+          "logprob": -12.640625,
           "text": "<image>"
         },
         {
@@ -11257,87 +10842,297 @@
         },
         {
           "id": 32000,
-          "logprob": -12.5078125,
+          "logprob": -12.4375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.09375,
+          "logprob": -11.8828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.8515625,
+          "logprob": -12.15625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8203125,
+          "logprob": -11.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.734375,
+          "logprob": -13.0,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2578125,
+          "logprob": -12.0703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.28125,
+          "logprob": -11.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.859375,
+          "logprob": -13.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.703125,
+          "logprob": -12.1484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5703125,
+          "logprob": -13.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.59375,
+          "logprob": -12.203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.890625,
+          "logprob": -11.9375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3359375,
+          "logprob": -12.1796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0625,
+          "logprob": -10.7421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.53125,
+          "logprob": -10.8203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7109375,
+          "logprob": -12.375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.7578125,
+          "logprob": -11.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.25,
           "text": "<image>"
         },
         {
@@ -11347,17 +11142,47 @@
         },
         {
           "id": 32000,
-          "logprob": -10.75,
+          "logprob": -11.8515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.2265625,
+          "logprob": -13.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9296875,
+          "logprob": -10.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1328125,
           "text": "<image>"
         },
         {
@@ -11367,7 +11192,12 @@
         },
         {
           "id": 32000,
-          "logprob": -10.5390625,
+          "logprob": -14.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0546875,
           "text": "<image>"
         },
         {
@@ -11375,6 +11205,56 @@
           "logprob": -11.6796875,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -11.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.234375,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -12.5390625,
@@ -11382,7 +11262,127 @@
         },
         {
           "id": 32000,
-          "logprob": -16.953125,
+          "logprob": -15.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.0,
           "text": "<image>"
         },
         {
@@ -11392,12 +11392,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.3359375,
+          "logprob": -11.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.671875,
+          "logprob": -11.6796875,
           "text": "<image>"
         },
         {
@@ -11412,12 +11412,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8671875,
+          "logprob": -11.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.484375,
+          "logprob": -11.4765625,
           "text": "<image>"
         },
         {
@@ -11427,12 +11427,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.6640625,
+          "logprob": -12.6875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4296875,
+          "logprob": -13.421875,
           "text": "<image>"
         },
         {
@@ -11447,7 +11447,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.3046875,
+          "logprob": -12.3125,
           "text": "<image>"
         },
         {
@@ -11457,17 +11457,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.34375,
+          "logprob": -13.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0703125,
+          "logprob": -13.0625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.234375,
+          "logprob": -11.2421875,
           "text": "<image>"
         },
         {
@@ -11477,37 +11477,37 @@
         },
         {
           "id": 32000,
-          "logprob": -13.4921875,
+          "logprob": -13.5,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.7578125,
+          "logprob": -10.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.53125,
+          "logprob": -13.5390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6953125,
+          "logprob": -12.703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.203125,
+          "logprob": -13.1953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -17.078125,
+          "logprob": -17.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.25,
+          "logprob": -15.0234375,
           "text": "<image>"
         },
         {
@@ -11517,17 +11517,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.140625,
+          "logprob": -12.125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5234375,
+          "logprob": -11.515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.0625,
+          "logprob": -16.125,
           "text": "<image>"
         },
         {
@@ -11537,17 +11537,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.6015625,
+          "logprob": -11.609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.5859375,
+          "logprob": -14.6015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.890625,
+          "logprob": -11.8984375,
           "text": "<image>"
         },
         {
@@ -11562,22 +11562,22 @@
         },
         {
           "id": 32000,
-          "logprob": -11.7734375,
+          "logprob": -11.78125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9921875,
+          "logprob": -13.0,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9375,
+          "logprob": -12.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9921875,
+          "logprob": -12.0,
           "text": "<image>"
         },
         {
@@ -11587,7 +11587,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.7578125,
+          "logprob": -11.7734375,
           "text": "<image>"
         },
         {
@@ -11597,7 +11597,7 @@
         },
         {
           "id": 32000,
-          "logprob": -14.328125,
+          "logprob": -14.3203125,
           "text": "<image>"
         },
         {
@@ -11607,22 +11607,22 @@
         },
         {
           "id": 32000,
-          "logprob": -13.84375,
+          "logprob": -13.875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.2109375,
+          "logprob": -14.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.390625,
+          "logprob": -12.375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.953125,
+          "logprob": -13.9375,
           "text": "<image>"
         },
         {
@@ -11632,27 +11632,27 @@
         },
         {
           "id": 32000,
-          "logprob": -14.6796875,
+          "logprob": -14.609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.84375,
+          "logprob": -11.8359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.65625,
+          "logprob": -10.6484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0078125,
+          "logprob": -12.0,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0859375,
+          "logprob": -11.0703125,
           "text": "<image>"
         },
         {
@@ -11672,7 +11672,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.6875,
+          "logprob": -12.6640625,
           "text": "<image>"
         },
         {
@@ -11687,22 +11687,22 @@
         },
         {
           "id": 32000,
-          "logprob": -16.109375,
+          "logprob": -16.09375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.984375,
+          "logprob": -11.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.40625,
+          "logprob": -13.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.375,
+          "logprob": -13.453125,
           "text": "<image>"
         },
         {
@@ -11712,57 +11712,57 @@
         },
         {
           "id": 32000,
-          "logprob": -12.921875,
+          "logprob": -12.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.859375,
+          "logprob": -14.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3671875,
+          "logprob": -13.375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0546875,
+          "logprob": -12.0703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.21875,
+          "logprob": -15.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6953125,
+          "logprob": -11.6875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.5234375,
+          "logprob": -10.5390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.8984375,
+          "logprob": -14.703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.8515625,
+          "logprob": -13.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6796875,
+          "logprob": -11.6875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -17.4375,
+          "logprob": -17.453125,
           "text": "<image>"
         },
         {
@@ -11777,7 +11777,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8125,
+          "logprob": -11.7890625,
           "text": "<image>"
         },
         {
@@ -11792,12 +11792,12 @@
         },
         {
           "id": 32000,
-          "logprob": -14.2421875,
+          "logprob": -14.265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.7265625,
+          "logprob": -13.6953125,
           "text": "<image>"
         },
         {
@@ -11812,57 +11812,57 @@
         },
         {
           "id": 32000,
-          "logprob": -12.84375,
+          "logprob": -12.890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9140625,
+          "logprob": -11.9375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.7421875,
+          "logprob": -14.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.984375,
+          "logprob": -11.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.71875,
+          "logprob": -14.7265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.578125,
+          "logprob": -11.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.015625,
+          "logprob": -13.0,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.796875,
+          "logprob": -12.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0078125,
+          "logprob": -11.015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.46875,
+          "logprob": -16.421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9296875,
+          "logprob": -11.921875,
           "text": "<image>"
         },
         {
@@ -11872,7 +11872,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.8515625,
+          "logprob": -12.859375,
           "text": "<image>"
         },
         {
@@ -11887,12 +11887,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.140625,
+          "logprob": -12.1328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.359375,
+          "logprob": -13.3515625,
           "text": "<image>"
         },
         {
@@ -11907,27 +11907,27 @@
         },
         {
           "id": 32000,
-          "logprob": -11.578125,
+          "logprob": -11.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8203125,
+          "logprob": -12.84375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.984375,
+          "logprob": -11.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5078125,
+          "logprob": -12.515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.625,
+          "logprob": -12.6328125,
           "text": "<image>"
         },
         {
@@ -11942,7 +11942,7 @@
         },
         {
           "id": 32000,
-          "logprob": -14.0546875,
+          "logprob": -14.046875,
           "text": "<image>"
         },
         {
@@ -11952,7 +11952,7 @@
         },
         {
           "id": 32000,
-          "logprob": -17.21875,
+          "logprob": -17.203125,
           "text": "<image>"
         },
         {
@@ -11962,17 +11962,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.3671875,
+          "logprob": -13.375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.6171875,
+          "logprob": -14.609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9609375,
+          "logprob": -12.953125,
           "text": "<image>"
         },
         {
@@ -11982,12 +11982,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.7265625,
+          "logprob": -11.71875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.03125,
+          "logprob": -15.0234375,
           "text": "<image>"
         },
         {
@@ -11997,7 +11997,7 @@
         },
         {
           "id": 32000,
-          "logprob": -15.890625,
+          "logprob": -15.9296875,
           "text": "<image>"
         },
         {
@@ -12022,22 +12022,22 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0234375,
+          "logprob": -12.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7578125,
+          "logprob": -12.75,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.7578125,
+          "logprob": -14.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5,
+          "logprob": -11.484375,
           "text": "<image>"
         },
         {
@@ -12047,7 +12047,7 @@
         },
         {
           "id": 32000,
-          "logprob": -10.7265625,
+          "logprob": -10.734375,
           "text": "<image>"
         },
         {
@@ -12057,22 +12057,22 @@
         },
         {
           "id": 32000,
-          "logprob": -11.4140625,
+          "logprob": -11.421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9453125,
+          "logprob": -12.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1171875,
+          "logprob": -13.0625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0390625,
+          "logprob": -12.03125,
           "text": "<image>"
         },
         {
@@ -12082,7 +12082,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.234375,
+          "logprob": -12.2109375,
           "text": "<image>"
         },
         {
@@ -12097,7 +12097,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.0390625,
+          "logprob": -13.09375,
           "text": "<image>"
         },
         {
@@ -12107,7 +12107,7 @@
         },
         {
           "id": 32000,
-          "logprob": -15.265625,
+          "logprob": -15.2578125,
           "text": "<image>"
         },
         {
@@ -12120,2201 +12120,6 @@
           "logprob": -11.0546875,
           "text": "<image>"
         },
-        {
-          "id": 32000,
-          "logprob": -11.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -19.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -18.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -18.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.7734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -19.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0625,
-          "text": "<image>"
-        },
         {
           "id": 32000,
           "logprob": -11.4609375,
@@ -14322,841 +12127,7 @@
         },
         {
           "id": 32000,
-          "logprob": -14.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.1953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5,
-          "text": "<image>"
-        },
-        {
-          "id": 2418,
-          "logprob": -19.0625,
-          "text": "Can"
-        },
-        {
-          "id": 368,
-          "logprob": -0.19726562,
-          "text": "you"
-        },
-        {
-          "id": 1912,
-          "logprob": -1.4990234,
-          "text": "tell"
-        },
-        {
-          "id": 528,
-          "logprob": -0.31152344,
-          "text": "me"
-        },
-        {
-          "id": 264,
-          "logprob": -2.6367188,
-          "text": "a"
-        },
-        {
-          "id": 1215,
-          "logprob": -9.1015625,
-          "text": "very"
-        },
-        {
-          "id": 2485,
-          "logprob": -0.9941406,
-          "text": "short"
-        },
-        {
-          "id": 2838,
-          "logprob": -0.46118164,
-          "text": "story"
-        },
-        {
-          "id": 2818,
-          "logprob": -3.3183594,
-          "text": "based"
-        },
-        {
-          "id": 356,
-          "logprob": -0.029129028,
-          "text": "on"
-        },
-        {
-          "id": 272,
-          "logprob": -0.9902344,
-          "text": "the"
-        },
-        {
-          "id": 3469,
-          "logprob": -0.29052734,
-          "text": "image"
-        },
-        {
-          "id": 28804,
-          "logprob": -0.43188477,
-          "text": "?"
-        }
-      ],
-      "seed": null,
-      "tokens": [
-        {
-          "id": 13,
-          "logprob": -0.0076828003,
-          "special": false,
-          "text": "\n"
-        },
-        {
-          "id": 13,
-          "logprob": -0.20092773,
-          "special": false,
-          "text": "\n"
-        },
-        {
-          "id": 16114,
-          "logprob": -1.2587891,
-          "special": false,
-          "text": "Once"
-        },
-        {
-          "id": 3714,
-          "logprob": -0.20861816,
-          "special": false,
-          "text": " upon"
-        },
-        {
-          "id": 264,
-          "logprob": -0.0017719269,
-          "special": false,
-          "text": " a"
-        },
-        {
-          "id": 727,
-          "logprob": -0.011909485,
-          "special": false,
-          "text": " time"
-        },
-        {
-          "id": 28725,
-          "logprob": -0.17529297,
-          "special": false,
-          "text": ","
-        },
-        {
-          "id": 736,
-          "logprob": -0.9082031,
-          "special": false,
-          "text": " there"
-        },
-        {
-          "id": 403,
-          "logprob": -0.057525635,
-          "special": false,
-          "text": " was"
-        },
-        {
-          "id": 264,
-          "logprob": -0.009651184,
-          "special": false,
-          "text": " a"
-        }
-      ],
-      "top_tokens": null
-    },
-    "generated_text": "\n\nOnce upon a time, there was a"
-  },
-  {
-    "details": {
-      "best_of_sequences": null,
-      "finish_reason": "length",
-      "generated_tokens": 10,
-      "prefill": [
-        {
-          "id": 1,
-          "logprob": null,
-          "text": "<s>"
-        },
-        {
-          "id": 1247,
-          "logprob": -2.3886719,
-          "text": "User"
-        },
-        {
-          "id": 28747,
-          "logprob": -12.328125,
-          "text": ":"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.7109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -18.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -9.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.7734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.7421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -18.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.7109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.2578125,
+          "logprob": -11.8046875,
           "text": "<image>"
         },
         {
@@ -15164,2156 +12135,11 @@
           "logprob": -11.140625,
           "text": "<image>"
         },
-        {
-          "id": 32000,
-          "logprob": -13.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.7890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.3671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.7109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.640625,
-          "text": "<image>"
-        },
         {
           "id": 32000,
           "logprob": -10.6875,
           "text": "<image>"
         },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -9.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.4609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.7421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.7890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.1875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5859375,
-          "text": "<image>"
-        },
         {
           "id": 32000,
           "logprob": -12.0546875,
@@ -17321,557 +12147,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.7890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -9.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7265625,
+          "logprob": -17.984375,
           "text": "<image>"
         },
         {
@@ -17879,1251 +12155,6 @@
           "logprob": -12.4140625,
           "text": "<image>"
         },
-        {
-          "id": 32000,
-          "logprob": -11.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -9.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.7890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -18.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.46875,
-          "text": "<image>"
-        },
         {
           "id": 32000,
           "logprob": -14.703125,
@@ -19131,122 +12162,7 @@
         },
         {
           "id": 32000,
-          "logprob": -14.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
+          "logprob": -11.796875,
           "text": "<image>"
         },
         {
@@ -19256,207 +12172,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.328125,
+          "logprob": -11.7109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6484375,
+          "logprob": -12.828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7578125,
+          "logprob": -12.8359375,
           "text": "<image>"
         },
         {
@@ -19466,237 +12192,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.4765625,
+          "logprob": -13.5390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6328125,
+          "logprob": -16.21875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.625,
+          "logprob": -16.84375,
           "text": "<image>"
         },
         {
@@ -19706,947 +12212,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.3515625,
+          "logprob": -14.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.7890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0234375,
+          "logprob": -11.71875,
           "text": "<image>"
         },
         {
@@ -20656,887 +12227,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.4609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.7890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7890625,
+          "logprob": -12.984375,
           "text": "<image>"
         },
         {
@@ -21546,32 +12237,112 @@
         },
         {
           "id": 32000,
-          "logprob": -13.4140625,
+          "logprob": -12.21875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.96875,
+          "logprob": -17.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1171875,
+          "logprob": -12.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5390625,
+          "logprob": -15.046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.609375,
+          "logprob": -16.71875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6484375,
+          "logprob": -11.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5625,
           "text": "<image>"
         },
         {
@@ -21581,32 +12352,1092 @@
         },
         {
           "id": 32000,
-          "logprob": -12.9921875,
+          "logprob": -11.9453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4453125,
+          "logprob": -11.8359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.9140625,
+          "logprob": -11.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.7109375,
+          "logprob": -12.875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.625,
+          "logprob": -11.9609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.765625,
+          "logprob": -13.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -19.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -18.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.0859375,
           "text": "<image>"
         },
         {
@@ -21619,6641 +13450,16 @@
           "logprob": -14.53125,
           "text": "<image>"
         },
-        {
-          "id": 32000,
-          "logprob": -13.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9296875,
-          "text": "<image>"
-        },
         {
           "id": 32000,
           "logprob": -12.359375,
           "text": "<image>"
         },
-        {
-          "id": 32000,
-          "logprob": -11.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.7109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1328125,
-          "text": "<image>"
-        },
         {
           "id": 32000,
           "logprob": -12.0625,
           "text": "<image>"
         },
-        {
-          "id": 32000,
-          "logprob": -11.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.1875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.7109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -9.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.7734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -18.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.7109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -9.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.7421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -19.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -18.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0390625,
-          "text": "<image>"
-        },
         {
           "id": 32000,
           "logprob": -13.640625,
@@ -28261,42 +13467,42 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8359375,
+          "logprob": -11.8203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1015625,
+          "logprob": -13.046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3515625,
+          "logprob": -13.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.578125,
+          "logprob": -12.625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.25,
+          "logprob": -11.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.15625,
+          "logprob": -14.2109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7265625,
+          "logprob": -11.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.90625,
+          "logprob": -13.8515625,
           "text": "<image>"
         },
         {
@@ -28306,12 +13512,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.6953125,
+          "logprob": -11.703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.5625,
+          "logprob": -10.65625,
           "text": "<image>"
         },
         {
@@ -28321,12 +13527,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.2265625,
+          "logprob": -12.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9296875,
+          "logprob": -11.8984375,
           "text": "<image>"
         },
         {
@@ -28336,12 +13542,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.875,
+          "logprob": -12.890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4453125,
+          "logprob": -11.453125,
           "text": "<image>"
         },
         {
@@ -28349,6 +13555,21 @@
           "logprob": -14.1796875,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -11.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.125,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -11.9453125,
@@ -28356,22 +13577,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5078125,
+          "logprob": -12.53125,
           "text": "<image>"
         },
         {
@@ -28381,27 +13587,27 @@
         },
         {
           "id": 32000,
-          "logprob": -12.2421875,
+          "logprob": -12.2265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3203125,
+          "logprob": -12.3046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.546875,
+          "logprob": -13.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.671875,
+          "logprob": -16.6875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.953125,
+          "logprob": -11.96875,
           "text": "<image>"
         },
         {
@@ -28411,7 +13617,7 @@
         },
         {
           "id": 32000,
-          "logprob": -14.9453125,
+          "logprob": -14.9140625,
           "text": "<image>"
         },
         {
@@ -28421,37 +13627,67 @@
         },
         {
           "id": 32000,
-          "logprob": -15.203125,
+          "logprob": -15.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.6953125,
+          "logprob": -14.6796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0234375,
+          "logprob": -12.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9453125,
+          "logprob": -12.9375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1796875,
+          "logprob": -12.2109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.75,
+          "logprob": -12.7421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3671875,
+          "logprob": -13.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6171875,
           "text": "<image>"
         },
         {
@@ -28461,42 +13697,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.4765625,
+          "logprob": -12.7109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3125,
+          "logprob": -13.2578125,
           "text": "<image>"
         },
         {
@@ -28506,12 +13712,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.484375,
+          "logprob": -13.5390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2734375,
+          "logprob": -13.265625,
           "text": "<image>"
         },
         {
@@ -28526,17 +13732,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.6796875,
+          "logprob": -11.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.53125,
+          "logprob": -16.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1484375,
+          "logprob": -14.171875,
           "text": "<image>"
         },
         {
@@ -28546,17 +13752,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8125,
+          "logprob": -11.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.03125,
+          "logprob": -15.1171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.484375,
+          "logprob": -15.4453125,
           "text": "<image>"
         },
         {
@@ -28571,37 +13777,37 @@
         },
         {
           "id": 32000,
-          "logprob": -10.6953125,
+          "logprob": -10.703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1640625,
+          "logprob": -11.15625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.84375,
+          "logprob": -12.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5625,
+          "logprob": -12.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7421875,
+          "logprob": -12.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -18.0,
+          "logprob": -17.953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5859375,
+          "logprob": -11.59375,
           "text": "<image>"
         },
         {
@@ -28616,12 +13822,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.6796875,
+          "logprob": -11.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.71875,
+          "logprob": -12.8515625,
           "text": "<image>"
         },
         {
@@ -28631,27 +13837,27 @@
         },
         {
           "id": 32000,
-          "logprob": -12.1875,
+          "logprob": -12.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7578125,
+          "logprob": -12.7265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.125,
+          "logprob": -15.140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.9140625,
+          "logprob": -14.9375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.546875,
+          "logprob": -15.6875,
           "text": "<image>"
         },
         {
@@ -28661,22 +13867,22 @@
         },
         {
           "id": 32000,
-          "logprob": -14.109375,
+          "logprob": -14.125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.234375,
+          "logprob": -13.2265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.15625,
+          "logprob": -13.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.734375,
+          "logprob": -12.703125,
           "text": "<image>"
         },
         {
@@ -28686,7 +13892,7 @@
         },
         {
           "id": 32000,
-          "logprob": -14.7734375,
+          "logprob": -14.765625,
           "text": "<image>"
         },
         {
@@ -28696,12 +13902,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.7421875,
+          "logprob": -11.71875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.921875,
+          "logprob": -11.9296875,
           "text": "<image>"
         },
         {
@@ -28711,17 +13917,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.6015625,
+          "logprob": -13.6328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4375,
+          "logprob": -11.4296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5,
+          "logprob": -13.5078125,
           "text": "<image>"
         },
         {
@@ -28736,27 +13942,27 @@
         },
         {
           "id": 32000,
-          "logprob": -12.1328125,
+          "logprob": -12.15625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.90625,
+          "logprob": -13.1171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.578125,
+          "logprob": -12.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3359375,
+          "logprob": -13.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7265625,
+          "logprob": -11.734375,
           "text": "<image>"
         },
         {
@@ -28764,491 +13970,11 @@
           "logprob": -10.953125,
           "text": "<image>"
         },
-        {
-          "id": 32000,
-          "logprob": -12.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -19.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.796875,
-          "text": "<image>"
-        },
         {
           "id": 32000,
           "logprob": -12.75,
           "text": "<image>"
         },
-        {
-          "id": 32000,
-          "logprob": -14.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.1953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.515625,
-          "text": "<image>"
-        },
         {
           "id": 32000,
           "logprob": -12.7734375,
@@ -29256,7 +13982,22 @@
         },
         {
           "id": 32000,
-          "logprob": -12.4609375,
+          "logprob": -13.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5390625,
           "text": "<image>"
         },
         {
@@ -29266,87 +14007,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.171875,
+          "logprob": -19.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8828125,
+          "logprob": -12.4765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8828125,
+          "logprob": -11.609375,
           "text": "<image>"
         },
         {
@@ -29359,6 +14030,541 @@
           "logprob": -13.3203125,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -12.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.359375,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -13.2109375,
@@ -29366,7 +14572,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.9765625,
+          "logprob": -11.984375,
           "text": "<image>"
         },
         {
@@ -29376,12 +14582,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0078125,
+          "logprob": -12.015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0390625,
+          "logprob": -11.03125,
           "text": "<image>"
         },
         {
@@ -29396,17 +14602,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.8203125,
+          "logprob": -13.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5078125,
+          "logprob": -13.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.734375,
+          "logprob": -11.7265625,
           "text": "<image>"
         },
         {
@@ -29416,22 +14622,22 @@
         },
         {
           "id": 32000,
-          "logprob": -17.3125,
+          "logprob": -16.84375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5234375,
+          "logprob": -12.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -17.625,
+          "logprob": -17.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9296875,
+          "logprob": -11.9453125,
           "text": "<image>"
         },
         {
@@ -29441,17 +14647,17 @@
         },
         {
           "id": 32000,
-          "logprob": -15.9140625,
+          "logprob": -16.0,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.65625,
+          "logprob": -16.625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5,
+          "logprob": -12.4921875,
           "text": "<image>"
         },
         {
@@ -29461,17 +14667,17 @@
         },
         {
           "id": 368,
-          "logprob": -0.19726562,
+          "logprob": -0.19604492,
           "text": "you"
         },
         {
           "id": 1912,
-          "logprob": -1.4990234,
+          "logprob": -1.5058594,
           "text": "tell"
         },
         {
           "id": 528,
-          "logprob": -0.31152344,
+          "logprob": -0.31030273,
           "text": "me"
         },
         {
@@ -29486,22 +14692,22 @@
         },
         {
           "id": 2485,
-          "logprob": -0.9941406,
+          "logprob": -0.9975586,
           "text": "short"
         },
         {
           "id": 2838,
-          "logprob": -0.46118164,
+          "logprob": -0.4633789,
           "text": "story"
         },
         {
           "id": 2818,
-          "logprob": -3.3183594,
+          "logprob": -3.3144531,
           "text": "based"
         },
         {
           "id": 356,
-          "logprob": -0.029129028,
+          "logprob": -0.029037476,
           "text": "on"
         },
         {
@@ -29511,12 +14717,12 @@
         },
         {
           "id": 3469,
-          "logprob": -0.29052734,
+          "logprob": -0.2890625,
           "text": "image"
         },
         {
           "id": 28804,
-          "logprob": -0.43188477,
+          "logprob": -0.42895508,
           "text": "?"
         }
       ],
@@ -29524,13 +14730,13 @@
       "tokens": [
         {
           "id": 13,
-          "logprob": -0.0076828003,
+          "logprob": -0.007621765,
           "special": false,
           "text": "\n"
         },
         {
           "id": 13,
-          "logprob": -0.19958496,
+          "logprob": -0.20812988,
           "special": false,
           "text": "\n"
         },
@@ -29542,37 +14748,37 @@
         },
         {
           "id": 3714,
-          "logprob": -0.20861816,
+          "logprob": -0.20825195,
           "special": false,
           "text": " upon"
         },
         {
           "id": 264,
-          "logprob": -0.0017719269,
+          "logprob": -0.0017709732,
           "special": false,
           "text": " a"
         },
         {
           "id": 727,
-          "logprob": -0.011749268,
+          "logprob": -0.011932373,
           "special": false,
           "text": " time"
         },
         {
           "id": 28725,
-          "logprob": -0.17529297,
+          "logprob": -0.17297363,
           "special": false,
           "text": ","
         },
         {
           "id": 736,
-          "logprob": -0.9086914,
+          "logprob": -0.9057617,
           "special": false,
           "text": " there"
         },
         {
           "id": 403,
-          "logprob": -0.056732178,
+          "logprob": -0.05758667,
           "special": false,
           "text": " was"
         },
@@ -29600,7 +14806,7 @@
         },
         {
           "id": 1247,
-          "logprob": -2.3886719,
+          "logprob": -2.390625,
           "text": "User"
         },
         {
@@ -29615,12 +14821,12 @@
         },
         {
           "id": 32000,
-          "logprob": -10.671875,
+          "logprob": -10.6953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.7109375,
+          "logprob": -15.828125,
           "text": "<image>"
         },
         {
@@ -29630,82 +14836,82 @@
         },
         {
           "id": 32000,
-          "logprob": -10.0234375,
+          "logprob": -10.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.1328125,
+          "logprob": -10.140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.421875,
+          "logprob": -10.3125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.90625,
+          "logprob": -12.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.59375,
+          "logprob": -15.4921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.828125,
+          "logprob": -13.8671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.390625,
+          "logprob": -11.3671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.1171875,
+          "logprob": -10.1015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.1640625,
+          "logprob": -10.1796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.234375,
+          "logprob": -10.2421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.3984375,
+          "logprob": -10.4609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.015625,
+          "logprob": -14.34375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0859375,
+          "logprob": -13.1796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2734375,
+          "logprob": -13.328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.359375,
+          "logprob": -14.3515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0390625,
+          "logprob": -11.0546875,
           "text": "<image>"
         },
         {
@@ -29715,12 +14921,12 @@
         },
         {
           "id": 32000,
-          "logprob": -10.5234375,
+          "logprob": -10.515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.4765625,
+          "logprob": -10.4453125,
           "text": "<image>"
         },
         {
@@ -29730,22 +14936,22 @@
         },
         {
           "id": 32000,
-          "logprob": -13.6171875,
+          "logprob": -13.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.359375,
+          "logprob": -11.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.8359375,
+          "logprob": -10.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -17.34375,
+          "logprob": -17.234375,
           "text": "<image>"
         },
         {
@@ -29760,12 +14966,12 @@
         },
         {
           "id": 32000,
-          "logprob": -10.640625,
+          "logprob": -10.6484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -18.390625,
+          "logprob": -17.984375,
           "text": "<image>"
         },
         {
@@ -29775,17 +14981,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.5625,
+          "logprob": -11.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -9.875,
+          "logprob": -9.8671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.7734375,
+          "logprob": -10.7578125,
           "text": "<image>"
         },
         {
@@ -29795,32 +15001,32 @@
         },
         {
           "id": 32000,
-          "logprob": -10.96875,
+          "logprob": -10.984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.609375,
+          "logprob": -10.625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.09375,
+          "logprob": -11.109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.5078125,
+          "logprob": -10.5,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4453125,
+          "logprob": -11.4609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.59375,
+          "logprob": -13.09375,
           "text": "<image>"
         },
         {
@@ -29830,132 +15036,87 @@
         },
         {
           "id": 32000,
-          "logprob": -10.5625,
+          "logprob": -10.5390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.640625,
+          "logprob": -10.625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9765625,
+          "logprob": -10.9296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.765625,
+          "logprob": -11.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3671875,
+          "logprob": -12.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0234375,
+          "logprob": -12.125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.59375,
+          "logprob": -10.578125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.7421875,
+          "logprob": -10.7265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0625,
+          "logprob": -11.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.3828125,
+          "logprob": -10.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.171875,
+          "logprob": -11.1796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0234375,
+          "logprob": -11.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -18.40625,
+          "logprob": -18.703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9921875,
+          "logprob": -11.015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.7109375,
+          "logprob": -15.6796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.15625,
+          "logprob": -12.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.140625,
+          "logprob": -10.4140625,
           "text": "<image>"
         },
         {
@@ -29965,12 +15126,42 @@
         },
         {
           "id": 32000,
-          "logprob": -10.7890625,
+          "logprob": -11.8046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4296875,
+          "logprob": -10.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.140625,
           "text": "<image>"
         },
         {
@@ -29980,12 +15171,27 @@
         },
         {
           "id": 32000,
-          "logprob": -10.796875,
+          "logprob": -10.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1640625,
+          "logprob": -11.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.15625,
           "text": "<image>"
         },
         {
@@ -29995,27 +15201,27 @@
         },
         {
           "id": 32000,
-          "logprob": -15.4453125,
+          "logprob": -15.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.2109375,
+          "logprob": -10.2265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.09375,
+          "logprob": -11.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.6796875,
+          "logprob": -14.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.3671875,
+          "logprob": -10.375,
           "text": "<image>"
         },
         {
@@ -30030,22 +15236,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.484375,
+          "logprob": -12.4921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.09375,
+          "logprob": -12.1015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.96875,
+          "logprob": -11.0859375,
           "text": "<image>"
         },
         {
@@ -30053,6 +15254,11 @@
           "logprob": -10.9765625,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -10.921875,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -11.0234375,
@@ -30068,11 +15274,6 @@
           "logprob": -11.1484375,
           "text": "<image>"
         },
-        {
-          "id": 32000,
-          "logprob": -10.703125,
-          "text": "<image>"
-        },
         {
           "id": 32000,
           "logprob": -10.71875,
@@ -30080,17 +15281,22 @@
         },
         {
           "id": 32000,
-          "logprob": -11.6875,
+          "logprob": -10.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.8984375,
+          "logprob": -11.6796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2890625,
+          "logprob": -10.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.375,
           "text": "<image>"
         },
         {
@@ -30100,12 +15306,12 @@
         },
         {
           "id": 32000,
-          "logprob": -10.640625,
+          "logprob": -10.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.7109375,
+          "logprob": -10.7265625,
           "text": "<image>"
         },
         {
@@ -30120,17 +15326,17 @@
         },
         {
           "id": 32000,
-          "logprob": -10.6875,
+          "logprob": -10.6796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.84375,
+          "logprob": -11.828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5078125,
+          "logprob": -11.5390625,
           "text": "<image>"
         },
         {
@@ -30140,22 +15346,22 @@
         },
         {
           "id": 32000,
-          "logprob": -10.9609375,
+          "logprob": -10.9765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4921875,
+          "logprob": -12.5,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5546875,
+          "logprob": -11.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.2265625,
+          "logprob": -11.2421875,
           "text": "<image>"
         },
         {
@@ -30165,7 +15371,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.21875,
+          "logprob": -11.2265625,
           "text": "<image>"
         },
         {
@@ -30175,52 +15381,52 @@
         },
         {
           "id": 32000,
-          "logprob": -11.0,
+          "logprob": -10.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0234375,
+          "logprob": -12.046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.09375,
+          "logprob": -11.109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.046875,
+          "logprob": -14.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.921875,
+          "logprob": -10.8984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9609375,
+          "logprob": -10.953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9140625,
+          "logprob": -10.9296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8125,
+          "logprob": -11.78125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.140625,
+          "logprob": -15.1484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9609375,
+          "logprob": -10.96875,
           "text": "<image>"
         },
         {
@@ -30230,72 +15436,72 @@
         },
         {
           "id": 32000,
-          "logprob": -10.8828125,
+          "logprob": -10.890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9609375,
+          "logprob": -10.9453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.46875,
+          "logprob": -11.453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5234375,
+          "logprob": -13.09375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.328125,
+          "logprob": -12.484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.375,
+          "logprob": -11.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3515625,
+          "logprob": -12.3359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.578125,
+          "logprob": -11.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5078125,
+          "logprob": -12.5,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.640625,
+          "logprob": -12.6328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1953125,
+          "logprob": -11.203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9921875,
+          "logprob": -11.9765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.921875,
+          "logprob": -10.9296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.7578125,
+          "logprob": -10.7421875,
           "text": "<image>"
         },
         {
@@ -30305,12 +15511,12 @@
         },
         {
           "id": 32000,
-          "logprob": -15.015625,
+          "logprob": -15.0234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7734375,
+          "logprob": -12.75,
           "text": "<image>"
         },
         {
@@ -30325,27 +15531,27 @@
         },
         {
           "id": 32000,
-          "logprob": -11.2890625,
+          "logprob": -11.265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.34375,
+          "logprob": -13.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.6953125,
+          "logprob": -10.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.59375,
+          "logprob": -10.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6015625,
+          "logprob": -11.7578125,
           "text": "<image>"
         },
         {
@@ -30360,7 +15566,7 @@
         },
         {
           "id": 32000,
-          "logprob": -10.5390625,
+          "logprob": -10.5859375,
           "text": "<image>"
         },
         {
@@ -30370,12 +15576,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.9765625,
+          "logprob": -17.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4609375,
+          "logprob": -11.4765625,
           "text": "<image>"
         },
         {
@@ -30385,12 +15591,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.015625,
+          "logprob": -12.0078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.84375,
+          "logprob": -12.8359375,
           "text": "<image>"
         },
         {
@@ -30400,12 +15606,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.4375,
+          "logprob": -12.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.671875,
+          "logprob": -12.6875,
           "text": "<image>"
         },
         {
@@ -30415,57 +15621,27 @@
         },
         {
           "id": 32000,
-          "logprob": -12.875,
+          "logprob": -12.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2578125,
+          "logprob": -12.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.359375,
+          "logprob": -10.3359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.375,
+          "logprob": -13.46875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.7421875,
+          "logprob": -11.8671875,
           "text": "<image>"
         },
         {
@@ -30475,7 +15651,32 @@
         },
         {
           "id": 32000,
-          "logprob": -11.0390625,
+          "logprob": -11.0078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8203125,
           "text": "<image>"
         },
         {
@@ -30485,17 +15686,22 @@
         },
         {
           "id": 32000,
-          "logprob": -10.6171875,
+          "logprob": -10.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2421875,
+          "logprob": -10.6328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.8359375,
+          "logprob": -12.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8203125,
           "text": "<image>"
         },
         {
@@ -30515,127 +15721,62 @@
         },
         {
           "id": 32000,
-          "logprob": -11.0,
+          "logprob": -10.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7734375,
+          "logprob": -12.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2109375,
+          "logprob": -12.21875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4296875,
+          "logprob": -11.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2421875,
+          "logprob": -12.828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3046875,
+          "logprob": -11.328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.2890625,
+          "logprob": -10.296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.8203125,
+          "logprob": -10.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9140625,
+          "logprob": -11.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.2421875,
+          "logprob": -11.21875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.234375,
+          "logprob": -11.2109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4375,
+          "logprob": -11.5078125,
           "text": "<image>"
         },
         {
@@ -30645,27 +15786,12 @@
         },
         {
           "id": 32000,
-          "logprob": -14.203125,
+          "logprob": -10.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.59375,
+          "logprob": -10.3515625,
           "text": "<image>"
         },
         {
@@ -30675,102 +15801,22 @@
         },
         {
           "id": 32000,
-          "logprob": -10.90625,
+          "logprob": -10.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.2578125,
+          "logprob": -10.828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.921875,
+          "logprob": -11.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.890625,
+          "logprob": -11.9375,
           "text": "<image>"
         },
         {
@@ -30780,112 +15826,32 @@
         },
         {
           "id": 32000,
-          "logprob": -10.9609375,
+          "logprob": -12.3046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.7890625,
+          "logprob": -11.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4453125,
+          "logprob": -12.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.390625,
+          "logprob": -13.3984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1640625,
+          "logprob": -11.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0703125,
+          "logprob": -14.1875,
           "text": "<image>"
         },
         {
@@ -30895,72 +15861,162 @@
         },
         {
           "id": 32000,
-          "logprob": -12.859375,
+          "logprob": -12.421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1875,
+          "logprob": -10.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.171875,
+          "logprob": -10.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6640625,
+          "logprob": -10.6015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1953125,
+          "logprob": -10.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1328125,
+          "logprob": -11.2265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1875,
+          "logprob": -10.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9453125,
+          "logprob": -10.9765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8515625,
+          "logprob": -12.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.203125,
+          "logprob": -10.8671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1796875,
+          "logprob": -10.8515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9609375,
+          "logprob": -11.8984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.703125,
+          "logprob": -10.7421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8515625,
+          "logprob": -13.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1796875,
           "text": "<image>"
         },
         {
@@ -30968,11 +16024,161 @@
           "logprob": -11.75,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -10.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.640625,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -11.8359375,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -11.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.828125,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -12.1953125,
@@ -30980,32 +16186,32 @@
         },
         {
           "id": 32000,
-          "logprob": -12.5078125,
+          "logprob": -12.4921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.546875,
+          "logprob": -11.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.078125,
+          "logprob": -12.0859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.2421875,
+          "logprob": -11.265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6640625,
+          "logprob": -11.6953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2109375,
+          "logprob": -12.21875,
           "text": "<image>"
         },
         {
@@ -31025,27 +16231,7 @@
         },
         {
           "id": 32000,
-          "logprob": -15.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3046875,
+          "logprob": -15.03125,
           "text": "<image>"
         },
         {
@@ -31055,32 +16241,52 @@
         },
         {
           "id": 32000,
-          "logprob": -12.2265625,
+          "logprob": -11.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9453125,
+          "logprob": -13.1171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4921875,
+          "logprob": -12.296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.8828125,
+          "logprob": -12.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3125,
+          "logprob": -12.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8984375,
+          "logprob": -12.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9140625,
           "text": "<image>"
         },
         {
@@ -31090,12 +16296,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8125,
+          "logprob": -11.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.90625,
+          "logprob": -11.9140625,
           "text": "<image>"
         },
         {
@@ -31115,12 +16321,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.140625,
+          "logprob": -13.1484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9765625,
+          "logprob": -11.9921875,
           "text": "<image>"
         },
         {
@@ -31135,7 +16341,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.84375,
+          "logprob": -11.8359375,
           "text": "<image>"
         },
         {
@@ -31150,17 +16356,22 @@
         },
         {
           "id": 32000,
-          "logprob": -13.1953125,
+          "logprob": -13.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.875,
+          "logprob": -11.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.6015625,
+          "logprob": -13.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6484375,
           "text": "<image>"
         },
         {
@@ -31170,27 +16381,22 @@
         },
         {
           "id": 32000,
-          "logprob": -13.671875,
+          "logprob": -11.4609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5390625,
+          "logprob": -13.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.59375,
+          "logprob": -12.4375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5703125,
+          "logprob": -14.6328125,
           "text": "<image>"
         },
         {
@@ -31200,57 +16406,57 @@
         },
         {
           "id": 32000,
-          "logprob": -13.4140625,
+          "logprob": -13.3984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7890625,
+          "logprob": -11.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6328125,
+          "logprob": -11.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.4296875,
+          "logprob": -14.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.53125,
+          "logprob": -13.5390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.515625,
+          "logprob": -14.5078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7265625,
+          "logprob": -12.7109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.609375,
+          "logprob": -11.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.171875,
+          "logprob": -13.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.109375,
+          "logprob": -13.09375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8828125,
+          "logprob": -12.875,
           "text": "<image>"
         },
         {
@@ -31260,37 +16466,37 @@
         },
         {
           "id": 32000,
-          "logprob": -12.671875,
+          "logprob": -12.6484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.7109375,
+          "logprob": -13.6953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4296875,
+          "logprob": -12.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.296875,
+          "logprob": -12.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1796875,
+          "logprob": -13.109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2421875,
+          "logprob": -12.1796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.828125,
+          "logprob": -14.875,
           "text": "<image>"
         },
         {
@@ -31300,7 +16506,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.3359375,
+          "logprob": -11.34375,
           "text": "<image>"
         },
         {
@@ -31320,67 +16526,7 @@
         },
         {
           "id": 32000,
-          "logprob": -15.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0,
+          "logprob": -15.0859375,
           "text": "<image>"
         },
         {
@@ -31390,57 +16536,117 @@
         },
         {
           "id": 32000,
-          "logprob": -16.21875,
+          "logprob": -11.875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6015625,
+          "logprob": -13.25,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.796875,
+          "logprob": -13.5078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7265625,
+          "logprob": -13.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9453125,
+          "logprob": -14.265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9765625,
+          "logprob": -14.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.71875,
+          "logprob": -13.2421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.6953125,
+          "logprob": -12.28125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3359375,
+          "logprob": -13.8046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3203125,
+          "logprob": -12.703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6328125,
+          "logprob": -12.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.609375,
           "text": "<image>"
         },
         {
@@ -31450,17 +16656,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1875,
+          "logprob": -14.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.046875,
+          "logprob": -12.0234375,
           "text": "<image>"
         },
         {
@@ -31475,17 +16681,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.8828125,
+          "logprob": -12.890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.296875,
+          "logprob": -12.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2109375,
+          "logprob": -12.21875,
           "text": "<image>"
         },
         {
@@ -31495,27 +16701,27 @@
         },
         {
           "id": 32000,
-          "logprob": -15.1875,
+          "logprob": -15.1953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5390625,
+          "logprob": -11.546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.421875,
+          "logprob": -14.4375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2890625,
+          "logprob": -12.265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2265625,
+          "logprob": -12.21875,
           "text": "<image>"
         },
         {
@@ -31530,12 +16736,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.5859375,
+          "logprob": -13.5390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.859375,
+          "logprob": -11.8515625,
           "text": "<image>"
         },
         {
@@ -31545,12 +16751,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.1015625,
+          "logprob": -13.046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.9453125,
+          "logprob": -13.953125,
           "text": "<image>"
         },
         {
@@ -31565,47 +16771,47 @@
         },
         {
           "id": 32000,
-          "logprob": -12.734375,
+          "logprob": -12.7109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.203125,
+          "logprob": -12.1796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.59375,
+          "logprob": -12.578125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3984375,
+          "logprob": -11.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5,
+          "logprob": -13.4609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.765625,
+          "logprob": -11.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2265625,
+          "logprob": -13.21875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7578125,
+          "logprob": -11.75,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.3515625,
+          "logprob": -15.34375,
           "text": "<image>"
         },
         {
@@ -31620,22 +16826,42 @@
         },
         {
           "id": 32000,
-          "logprob": -13.3671875,
+          "logprob": -13.3515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.90625,
+          "logprob": -11.8828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5625,
+          "logprob": -12.578125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3203125,
+          "logprob": -13.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.125,
           "text": "<image>"
         },
         {
@@ -31645,27 +16871,7 @@
         },
         {
           "id": 32000,
-          "logprob": -10.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9921875,
+          "logprob": -11.984375,
           "text": "<image>"
         },
         {
@@ -31680,27 +16886,27 @@
         },
         {
           "id": 32000,
-          "logprob": -12.9765625,
+          "logprob": -12.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4921875,
+          "logprob": -12.484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0625,
+          "logprob": -12.046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1796875,
+          "logprob": -12.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3359375,
+          "logprob": -13.296875,
           "text": "<image>"
         },
         {
@@ -31710,32 +16916,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.375,
+          "logprob": -12.3671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.984375,
+          "logprob": -13.1328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9453125,
+          "logprob": -13.625,
           "text": "<image>"
         },
         {
@@ -31745,87 +16936,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.3203125,
+          "logprob": -11.75,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3125,
+          "logprob": -11.9140625,
           "text": "<image>"
         },
         {
@@ -31835,12 +16951,102 @@
         },
         {
           "id": 32000,
-          "logprob": -11.3984375,
+          "logprob": -12.3359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.84375,
+          "logprob": -12.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8359375,
           "text": "<image>"
         },
         {
@@ -31850,52 +17056,52 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6328125,
+          "logprob": -12.6171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.875,
+          "logprob": -13.84375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.109375,
+          "logprob": -12.09375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1171875,
+          "logprob": -12.140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4921875,
+          "logprob": -11.5078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.2890625,
+          "logprob": -11.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.15625,
+          "logprob": -12.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.59375,
+          "logprob": -13.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8046875,
+          "logprob": -11.7890625,
           "text": "<image>"
         },
         {
@@ -31905,22 +17111,22 @@
         },
         {
           "id": 32000,
-          "logprob": -13.1015625,
+          "logprob": -13.0625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2265625,
+          "logprob": -13.1875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2109375,
+          "logprob": -13.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4609375,
+          "logprob": -12.46875,
           "text": "<image>"
         },
         {
@@ -31930,12 +17136,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
+          "logprob": -12.6953125,
           "text": "<image>"
         },
         {
@@ -31943,6 +17144,11 @@
           "logprob": -11.6875,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -11.6953125,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -11.7109375,
@@ -31950,27 +17156,27 @@
         },
         {
           "id": 32000,
-          "logprob": -15.390625,
+          "logprob": -15.3359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.953125,
+          "logprob": -11.9453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.140625,
+          "logprob": -16.125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.4921875,
+          "logprob": -15.4609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.9296875,
+          "logprob": -13.921875,
           "text": "<image>"
         },
         {
@@ -31985,7 +17191,7 @@
         },
         {
           "id": 32000,
-          "logprob": -15.984375,
+          "logprob": -16.0,
           "text": "<image>"
         },
         {
@@ -31995,37 +17201,37 @@
         },
         {
           "id": 32000,
-          "logprob": -12.8671875,
+          "logprob": -12.828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4921875,
+          "logprob": -12.5078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7421875,
+          "logprob": -11.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1875,
+          "logprob": -14.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3515625,
+          "logprob": -11.34375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.71875,
+          "logprob": -11.7265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0625,
+          "logprob": -13.046875,
           "text": "<image>"
         },
         {
@@ -32035,12 +17241,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.421875,
+          "logprob": -12.4375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.2734375,
+          "logprob": -14.265625,
           "text": "<image>"
         },
         {
@@ -32050,77 +17256,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.390625,
+          "logprob": -12.6953125,
           "text": "<image>"
         },
         {
@@ -32130,27 +17266,97 @@
         },
         {
           "id": 32000,
-          "logprob": -13.078125,
+          "logprob": -12.984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4296875,
+          "logprob": -13.2890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.828125,
+          "logprob": -13.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8046875,
+          "logprob": -11.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6484375,
+          "logprob": -10.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.625,
           "text": "<image>"
         },
         {
@@ -32165,27 +17371,27 @@
         },
         {
           "id": 32000,
-          "logprob": -14.921875,
+          "logprob": -14.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.578125,
+          "logprob": -12.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.78125,
+          "logprob": -12.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3984375,
+          "logprob": -11.375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.0546875,
+          "logprob": -14.03125,
           "text": "<image>"
         },
         {
@@ -32205,12 +17411,12 @@
         },
         {
           "id": 32000,
-          "logprob": -14.5234375,
+          "logprob": -14.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.625,
+          "logprob": -12.609375,
           "text": "<image>"
         },
         {
@@ -32220,12 +17426,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.6015625,
+          "logprob": -13.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.28125,
+          "logprob": -13.2109375,
           "text": "<image>"
         },
         {
@@ -32245,17 +17451,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.0,
+          "logprob": -12.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6640625,
+          "logprob": -12.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.46875,
+          "logprob": -11.484375,
           "text": "<image>"
         },
         {
@@ -32265,12 +17471,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8828125,
+          "logprob": -11.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1015625,
+          "logprob": -13.078125,
           "text": "<image>"
         },
         {
@@ -32285,7 +17491,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.5234375,
+          "logprob": -11.546875,
           "text": "<image>"
         },
         {
@@ -32300,7 +17506,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.4921875,
+          "logprob": -12.5,
           "text": "<image>"
         },
         {
@@ -32315,122 +17521,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.7890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1484375,
+          "logprob": -12.078125,
           "text": "<image>"
         },
         {
@@ -32440,132 +17531,102 @@
         },
         {
           "id": 32000,
-          "logprob": -13.3828125,
+          "logprob": -13.2265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3203125,
+          "logprob": -14.8046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.34375,
+          "logprob": -12.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -11.8671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.140625,
+          "logprob": -13.1484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9921875,
+          "logprob": -12.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.15625,
+          "logprob": -10.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.84375,
+          "logprob": -14.6875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.21875,
+          "logprob": -15.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.46875,
+          "logprob": -15.453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1484375,
+          "logprob": -11.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.515625,
+          "logprob": -14.484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.84375,
+          "logprob": -12.4375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.015625,
+          "logprob": -16.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.28125,
+          "logprob": -14.875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6015625,
+          "logprob": -16.109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3984375,
+          "logprob": -13.2890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.375,
+          "logprob": -12.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5625,
+          "logprob": -11.8359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -9.9375,
+          "logprob": -12.9921875,
           "text": "<image>"
         },
         {
@@ -32575,17 +17636,162 @@
         },
         {
           "id": 32000,
-          "logprob": -12.203125,
+          "logprob": -12.1484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.2890625,
+          "logprob": -13.84375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1953125,
+          "logprob": -13.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -9.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.15625,
           "text": "<image>"
         },
         {
@@ -32598,21 +17804,6 @@
           "logprob": -12.28125,
           "text": "<image>"
         },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
         {
           "id": 32000,
           "logprob": -11.8984375,
@@ -32620,52 +17811,67 @@
         },
         {
           "id": 32000,
-          "logprob": -10.5859375,
+          "logprob": -14.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9921875,
+          "logprob": -14.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9921875,
+          "logprob": -11.8671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1015625,
+          "logprob": -10.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.5390625,
+          "logprob": -13.0,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.2578125,
+          "logprob": -12.8984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1171875,
+          "logprob": -12.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2421875,
+          "logprob": -15.765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5,
+          "logprob": -15.203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7265625,
+          "logprob": -14.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8359375,
           "text": "<image>"
         },
         {
@@ -32675,7 +17881,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.0,
+          "logprob": -11.0078125,
           "text": "<image>"
         },
         {
@@ -32685,22 +17891,22 @@
         },
         {
           "id": 32000,
-          "logprob": -13.28125,
+          "logprob": -13.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7734375,
+          "logprob": -11.7578125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0625,
+          "logprob": -13.09375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.671875,
+          "logprob": -13.6796875,
           "text": "<image>"
         },
         {
@@ -32710,7 +17916,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.3828125,
+          "logprob": -12.421875,
           "text": "<image>"
         },
         {
@@ -32720,22 +17926,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.25,
+          "logprob": -12.2890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.9453125,
+          "logprob": -11.921875,
           "text": "<image>"
         },
         {
@@ -32745,7 +17941,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.9375,
+          "logprob": -15.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
           "text": "<image>"
         },
         {
@@ -32755,37 +17961,37 @@
         },
         {
           "id": 32000,
-          "logprob": -11.3203125,
+          "logprob": -11.328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4921875,
+          "logprob": -11.5,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.359375,
+          "logprob": -11.3671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3359375,
+          "logprob": -11.3515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0546875,
+          "logprob": -12.0625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.359375,
+          "logprob": -12.3359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -9.6953125,
+          "logprob": -9.7109375,
           "text": "<image>"
         },
         {
@@ -32795,27 +18001,27 @@
         },
         {
           "id": 32000,
-          "logprob": -14.3203125,
+          "logprob": -14.2421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9609375,
+          "logprob": -11.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0859375,
+          "logprob": -12.09375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2109375,
+          "logprob": -12.1796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4921875,
+          "logprob": -11.5234375,
           "text": "<image>"
         },
         {
@@ -32830,7 +18036,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.0546875,
           "text": "<image>"
         },
         {
@@ -32845,22 +18051,22 @@
         },
         {
           "id": 32000,
-          "logprob": -11.9921875,
+          "logprob": -12.0,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2265625,
+          "logprob": -12.1953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9921875,
+          "logprob": -12.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6796875,
+          "logprob": -12.7109375,
           "text": "<image>"
         },
         {
@@ -32870,7 +18076,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.5703125,
+          "logprob": -13.5546875,
           "text": "<image>"
         },
         {
@@ -32885,17 +18091,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.9375,
+          "logprob": -11.953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.9453125,
+          "logprob": -13.8046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.984375,
+          "logprob": -11.9921875,
           "text": "<image>"
         },
         {
@@ -32905,17 +18111,17 @@
         },
         {
           "id": 32000,
-          "logprob": -10.03125,
+          "logprob": -10.015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7265625,
+          "logprob": -11.71875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.7890625,
+          "logprob": -13.8125,
           "text": "<image>"
         },
         {
@@ -32925,42 +18131,42 @@
         },
         {
           "id": 32000,
-          "logprob": -12.671875,
+          "logprob": -12.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.59375,
+          "logprob": -12.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1171875,
+          "logprob": -12.1015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4609375,
+          "logprob": -12.453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3046875,
+          "logprob": -12.2890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4765625,
+          "logprob": -12.46875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6328125,
+          "logprob": -12.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.03125,
+          "logprob": -12.015625,
           "text": "<image>"
         },
         {
@@ -32970,132 +18176,32 @@
         },
         {
           "id": 32000,
-          "logprob": -14.5078125,
+          "logprob": -14.421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2265625,
+          "logprob": -13.25,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.546875,
+          "logprob": -16.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.4765625,
+          "logprob": -14.484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.71875,
+          "logprob": -13.7421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.7890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.671875,
+          "logprob": -13.46875,
           "text": "<image>"
         },
         {
@@ -33105,57 +18211,97 @@
         },
         {
           "id": 32000,
-          "logprob": -13.40625,
+          "logprob": -12.8671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.265625,
+          "logprob": -12.4921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.015625,
+          "logprob": -13.2265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5390625,
+          "logprob": -13.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.0234375,
+          "logprob": -12.4375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1796875,
+          "logprob": -10.765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6875,
+          "logprob": -13.109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.9453125,
+          "logprob": -12.8515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6875,
+          "logprob": -14.7421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.78125,
+          "logprob": -12.84375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.9609375,
+          "logprob": -12.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6484375,
           "text": "<image>"
         },
         {
@@ -33165,27 +18311,87 @@
         },
         {
           "id": 32000,
-          "logprob": -16.140625,
+          "logprob": -13.4296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4609375,
+          "logprob": -13.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.7265625,
+          "logprob": -12.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.609375,
+          "logprob": -11.5234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.234375,
+          "logprob": -15.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.25,
           "text": "<image>"
         },
         {
@@ -33195,7 +18401,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.4921875,
+          "logprob": -13.5,
           "text": "<image>"
         },
         {
@@ -33220,12 +18426,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.15625,
+          "logprob": -13.1484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -17.046875,
+          "logprob": -17.0625,
           "text": "<image>"
         },
         {
@@ -33235,12 +18441,12 @@
         },
         {
           "id": 32000,
-          "logprob": -14.265625,
+          "logprob": -14.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6328125,
+          "logprob": -12.625,
           "text": "<image>"
         },
         {
@@ -33250,92 +18456,37 @@
         },
         {
           "id": 32000,
-          "logprob": -13.9375,
+          "logprob": -13.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0390625,
+          "logprob": -13.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.46875,
+          "logprob": -10.5,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1640625,
+          "logprob": -13.1953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.59375,
+          "logprob": -13.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.390625,
+          "logprob": -13.3828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8125,
+          "logprob": -14.0625,
           "text": "<image>"
         },
         {
@@ -33343,6 +18494,61 @@
           "logprob": -13.5234375,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -14.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.53125,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -11.4140625,
@@ -33350,17 +18556,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.46875,
+          "logprob": -11.4609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.09375,
+          "logprob": -13.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.34375,
+          "logprob": -12.3359375,
           "text": "<image>"
         },
         {
@@ -33370,32 +18576,32 @@
         },
         {
           "id": 32000,
-          "logprob": -12.7578125,
+          "logprob": -12.75,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.8359375,
+          "logprob": -14.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0859375,
+          "logprob": -13.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1640625,
+          "logprob": -14.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0625,
+          "logprob": -13.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.5078125,
+          "logprob": -14.4921875,
           "text": "<image>"
         },
         {
@@ -33405,7 +18611,7 @@
         },
         {
           "id": 32000,
-          "logprob": -14.6953125,
+          "logprob": -14.703125,
           "text": "<image>"
         },
         {
@@ -33415,32 +18621,32 @@
         },
         {
           "id": 32000,
-          "logprob": -16.328125,
+          "logprob": -16.3125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.9921875,
+          "logprob": -14.9765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.8515625,
+          "logprob": -13.8046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.40625,
+          "logprob": -15.375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4296875,
+          "logprob": -13.3984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.5859375,
+          "logprob": -14.578125,
           "text": "<image>"
         },
         {
@@ -33455,7 +18661,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.3359375,
+          "logprob": -13.328125,
           "text": "<image>"
         },
         {
@@ -33470,37 +18676,52 @@
         },
         {
           "id": 32000,
-          "logprob": -16.890625,
+          "logprob": -16.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.328125,
+          "logprob": -13.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.78125,
+          "logprob": -13.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.34375,
+          "logprob": -13.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4921875,
+          "logprob": -13.4765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4296875,
+          "logprob": -13.390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.28125,
+          "logprob": -13.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5,
           "text": "<image>"
         },
         {
@@ -33510,7 +18731,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.75,
+          "logprob": -13.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6796875,
           "text": "<image>"
         },
         {
@@ -33520,32 +18751,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.328125,
+          "logprob": -13.3203125,
           "text": "<image>"
         },
         {
@@ -33555,47 +18761,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.640625,
+          "logprob": -13.6171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.15625,
+          "logprob": -15.140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0078125,
+          "logprob": -13.21875,
           "text": "<image>"
         },
         {
@@ -33605,27 +18781,57 @@
         },
         {
           "id": 32000,
-          "logprob": -11.625,
+          "logprob": -12.0234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8671875,
+          "logprob": -11.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.578125,
+          "logprob": -11.3125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8125,
+          "logprob": -10.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.9453125,
+          "logprob": -13.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.9765625,
           "text": "<image>"
         },
         {
@@ -33635,17 +18841,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.9765625,
+          "logprob": -12.984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.828125,
+          "logprob": -13.8359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7265625,
+          "logprob": -12.75,
           "text": "<image>"
         },
         {
@@ -33660,12 +18866,12 @@
         },
         {
           "id": 32000,
-          "logprob": -14.7578125,
+          "logprob": -14.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.5234375,
+          "logprob": -14.5078125,
           "text": "<image>"
         },
         {
@@ -33675,12 +18881,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.0390625,
+          "logprob": -13.046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4453125,
+          "logprob": -13.453125,
           "text": "<image>"
         },
         {
@@ -33695,17 +18901,17 @@
         },
         {
           "id": 32000,
-          "logprob": -14.0,
+          "logprob": -13.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5390625,
+          "logprob": -13.4921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0859375,
+          "logprob": -11.9765625,
           "text": "<image>"
         },
         {
@@ -33715,17 +18921,37 @@
         },
         {
           "id": 32000,
-          "logprob": -15.953125,
+          "logprob": -15.9609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.2265625,
+          "logprob": -14.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1484375,
+          "logprob": -13.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.640625,
           "text": "<image>"
         },
         {
@@ -33735,32 +18961,47 @@
         },
         {
           "id": 32000,
-          "logprob": -12.6875,
+          "logprob": -13.625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.796875,
+          "logprob": -12.890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.65625,
+          "logprob": -12.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6796875,
+          "logprob": -12.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.65625,
+          "logprob": -12.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.90625,
+          "logprob": -14.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0078125,
           "text": "<image>"
         },
         {
@@ -33770,52 +19011,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.6171875,
+          "logprob": -15.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7890625,
+          "logprob": -12.921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.375,
+          "logprob": -12.40625,
           "text": "<image>"
         },
         {
@@ -33830,27 +19036,27 @@
         },
         {
           "id": 32000,
-          "logprob": -11.375,
+          "logprob": -11.3671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0625,
+          "logprob": -11.0703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3046875,
+          "logprob": -11.28125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0390625,
+          "logprob": -11.046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3515625,
+          "logprob": -13.3203125,
           "text": "<image>"
         },
         {
@@ -33875,12 +19081,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.8359375,
+          "logprob": -13.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0234375,
+          "logprob": -12.0859375,
           "text": "<image>"
         },
         {
@@ -33890,12 +19096,7 @@
         },
         {
           "id": 32000,
-          "logprob": -14.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0546875,
+          "logprob": -14.0703125,
           "text": "<image>"
         },
         {
@@ -33905,7 +19106,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.0625,
+          "logprob": -14.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0390625,
           "text": "<image>"
         },
         {
@@ -33915,42 +19121,42 @@
         },
         {
           "id": 32000,
-          "logprob": -14.46875,
+          "logprob": -14.453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.703125,
+          "logprob": -14.6875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.296875,
+          "logprob": -14.3046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.8828125,
+          "logprob": -14.8671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1796875,
+          "logprob": -14.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2265625,
+          "logprob": -13.203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1171875,
+          "logprob": -13.109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9375,
+          "logprob": -11.8515625,
           "text": "<image>"
         },
         {
@@ -33965,12 +19171,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.71875,
+          "logprob": -12.7109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.734375,
+          "logprob": -13.7109375,
           "text": "<image>"
         },
         {
@@ -33980,32 +19186,32 @@
         },
         {
           "id": 32000,
-          "logprob": -12.46875,
+          "logprob": -12.4765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.15625,
+          "logprob": -12.140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.375,
+          "logprob": -12.3828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8984375,
+          "logprob": -12.875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.6953125,
+          "logprob": -13.6484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0625,
+          "logprob": -13.0390625,
           "text": "<image>"
         },
         {
@@ -34015,12 +19221,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.6796875,
+          "logprob": -12.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3515625,
+          "logprob": -12.34375,
           "text": "<image>"
         },
         {
@@ -34030,37 +19236,37 @@
         },
         {
           "id": 32000,
-          "logprob": -12.46875,
+          "logprob": -12.453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.03125,
+          "logprob": -12.015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.96875,
+          "logprob": -11.9765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.4609375,
+          "logprob": -14.46875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.328125,
+          "logprob": -13.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6484375,
+          "logprob": -12.71875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.65625,
+          "logprob": -10.6484375,
           "text": "<image>"
         },
         {
@@ -34080,22 +19286,22 @@
         },
         {
           "id": 32000,
-          "logprob": -12.6328125,
+          "logprob": -12.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.265625,
+          "logprob": -11.2578125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.734375,
+          "logprob": -11.7265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.84375,
+          "logprob": -10.8515625,
           "text": "<image>"
         },
         {
@@ -34105,32 +19311,32 @@
         },
         {
           "id": 32000,
-          "logprob": -13.109375,
+          "logprob": -13.140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7421875,
+          "logprob": -12.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.09375,
+          "logprob": -12.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.109375,
+          "logprob": -13.1328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.671875,
+          "logprob": -12.65625,
           "text": "<image>"
         },
         {
@@ -34140,27 +19346,27 @@
         },
         {
           "id": 32000,
-          "logprob": -12.6796875,
+          "logprob": -12.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.109375,
+          "logprob": -12.0859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.625,
+          "logprob": -13.609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8515625,
+          "logprob": -12.8359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3046875,
+          "logprob": -12.3125,
           "text": "<image>"
         },
         {
@@ -34170,12 +19376,12 @@
         },
         {
           "id": 32000,
-          "logprob": -14.0703125,
+          "logprob": -14.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.0,
+          "logprob": -13.984375,
           "text": "<image>"
         },
         {
@@ -34185,7 +19391,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.2421875,
+          "logprob": -13.2109375,
           "text": "<image>"
         },
         {
@@ -34195,42 +19401,42 @@
         },
         {
           "id": 32000,
-          "logprob": -13.7109375,
+          "logprob": -13.703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6015625,
+          "logprob": -12.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.796875,
+          "logprob": -15.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.90625,
+          "logprob": -11.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.375,
+          "logprob": -12.3671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4765625,
+          "logprob": -11.46875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.015625,
+          "logprob": -12.0,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.328125,
+          "logprob": -11.3359375,
           "text": "<image>"
         },
         {
@@ -34245,17 +19451,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.8203125,
+          "logprob": -13.828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7578125,
+          "logprob": -11.75,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.484375,
+          "logprob": -11.46875,
           "text": "<image>"
         },
         {
@@ -34270,17 +19476,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.640625,
+          "logprob": -11.625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5859375,
+          "logprob": -11.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.2578125,
+          "logprob": -15.2265625,
           "text": "<image>"
         },
         {
@@ -34290,17 +19496,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.0078125,
+          "logprob": -13.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.3828125,
+          "logprob": -15.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.203125,
+          "logprob": -11.1875,
           "text": "<image>"
         },
         {
@@ -34310,7 +19516,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.0703125,
+          "logprob": -11.0625,
           "text": "<image>"
         },
         {
@@ -34320,7 +19526,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.4453125,
+          "logprob": -13.453125,
           "text": "<image>"
         },
         {
@@ -34330,27 +19536,27 @@
         },
         {
           "id": 32000,
-          "logprob": -11.5078125,
+          "logprob": -11.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.015625,
+          "logprob": -12.0078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.15625,
+          "logprob": -11.1875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8359375,
+          "logprob": -11.890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.359375,
+          "logprob": -11.3125,
           "text": "<image>"
         },
         {
@@ -34365,22 +19571,22 @@
         },
         {
           "id": 32000,
-          "logprob": -12.4453125,
+          "logprob": -12.4921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.015625,
+          "logprob": -12.0703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.375,
+          "logprob": -13.3671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2734375,
+          "logprob": -13.28125,
           "text": "<image>"
         },
         {
@@ -34390,22 +19596,22 @@
         },
         {
           "id": 32000,
-          "logprob": -13.046875,
+          "logprob": -13.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0859375,
+          "logprob": -13.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1640625,
+          "logprob": -12.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4765625,
+          "logprob": -13.484375,
           "text": "<image>"
         },
         {
@@ -34415,7 +19621,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.7265625,
+          "logprob": -13.703125,
           "text": "<image>"
         },
         {
@@ -34425,262 +19631,17 @@
         },
         {
           "id": 32000,
-          "logprob": -14.3359375,
+          "logprob": -14.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.71875,
+          "logprob": -12.7109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.640625,
+          "logprob": -14.28125,
           "text": "<image>"
         },
         {
@@ -34690,52 +19651,57 @@
         },
         {
           "id": 32000,
-          "logprob": -13.453125,
+          "logprob": -13.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.4375,
+          "logprob": -15.203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1171875,
+          "logprob": -13.28125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4296875,
+          "logprob": -12.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.515625,
+          "logprob": -14.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.21875,
+          "logprob": -13.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8671875,
+          "logprob": -12.1171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8515625,
+          "logprob": -14.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.8671875,
+          "logprob": -14.0625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.59375,
+          "logprob": -12.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.953125,
           "text": "<image>"
         },
         {
@@ -34745,12 +19711,57 @@
         },
         {
           "id": 32000,
-          "logprob": -12.1640625,
+          "logprob": -11.6953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2265625,
+          "logprob": -12.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5234375,
           "text": "<image>"
         },
         {
@@ -34760,32 +19771,27 @@
         },
         {
           "id": 32000,
-          "logprob": -12.1015625,
+          "logprob": -11.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0234375,
+          "logprob": -11.2578125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4140625,
+          "logprob": -13.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.828125,
+          "logprob": -11.921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1953125,
+          "logprob": -13.03125,
           "text": "<image>"
         },
         {
@@ -34795,7 +19801,207 @@
         },
         {
           "id": 32000,
-          "logprob": -11.1328125,
+          "logprob": -14.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.4296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.140625,
           "text": "<image>"
         },
         {
@@ -34820,7 +20026,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.1015625,
+          "logprob": -11.1171875,
           "text": "<image>"
         },
         {
@@ -34830,7 +20036,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.15625,
+          "logprob": -11.1484375,
           "text": "<image>"
         },
         {
@@ -34840,17 +20046,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.4296875,
+          "logprob": -11.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.765625,
+          "logprob": -12.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.09375,
+          "logprob": -13.0859375,
           "text": "<image>"
         },
         {
@@ -34860,22 +20066,22 @@
         },
         {
           "id": 32000,
-          "logprob": -12.96875,
+          "logprob": -12.9609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.328125,
+          "logprob": -12.34375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8359375,
+          "logprob": -12.84375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.9609375,
+          "logprob": -13.9765625,
           "text": "<image>"
         },
         {
@@ -34885,7 +20091,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.46875,
+          "logprob": -13.484375,
           "text": "<image>"
         },
         {
@@ -34895,22 +20101,22 @@
         },
         {
           "id": 32000,
-          "logprob": -13.59375,
+          "logprob": -13.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.40625,
+          "logprob": -13.3828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.234375,
+          "logprob": -14.25,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.265625,
+          "logprob": -14.2578125,
           "text": "<image>"
         },
         {
@@ -34920,127 +20126,127 @@
         },
         {
           "id": 32000,
-          "logprob": -13.65625,
+          "logprob": -13.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.2734375,
+          "logprob": -14.2578125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.09375,
+          "logprob": -13.8671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.890625,
+          "logprob": -12.921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.640625,
+          "logprob": -15.6484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0234375,
+          "logprob": -13.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3828125,
+          "logprob": -12.34375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.71875,
+          "logprob": -12.7265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.5234375,
+          "logprob": -14.515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7734375,
+          "logprob": -12.75,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3203125,
+          "logprob": -12.265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.609375,
+          "logprob": -13.546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1640625,
+          "logprob": -13.1328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.203125,
+          "logprob": -12.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1875,
+          "logprob": -12.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.59375,
+          "logprob": -11.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6875,
+          "logprob": -11.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1796875,
+          "logprob": -12.15625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6484375,
+          "logprob": -11.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.421875,
+          "logprob": -11.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.34375,
+          "logprob": -11.390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1015625,
+          "logprob": -11.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9765625,
+          "logprob": -11.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.015625,
+          "logprob": -12.0078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0,
+          "logprob": -12.984375,
           "text": "<image>"
         },
         {
@@ -35050,7 +20256,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.09375,
           "text": "<image>"
         },
         {
@@ -35070,12 +20276,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.65625,
+          "logprob": -11.6484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.84375,
+          "logprob": -12.8125,
           "text": "<image>"
         },
         {
@@ -35085,17 +20291,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.6953125,
+          "logprob": -11.6796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.578125,
+          "logprob": -11.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.6796875,
+          "logprob": -13.6953125,
           "text": "<image>"
         },
         {
@@ -35110,7 +20316,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0078125,
+          "logprob": -12.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.296875,
           "text": "<image>"
         },
         {
@@ -35120,17 +20331,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.234375,
+          "logprob": -13.84375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8671875,
+          "logprob": -12.875,
           "text": "<image>"
         },
         {
@@ -35140,17 +20346,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.5078125,
+          "logprob": -13.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.953125,
+          "logprob": -13.9609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5078125,
+          "logprob": -12.46875,
           "text": "<image>"
         },
         {
@@ -35160,232 +20366,17 @@
         },
         {
           "id": 32000,
-          "logprob": -14.25,
+          "logprob": -14.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9140625,
+          "logprob": -12.921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4375,
+          "logprob": -14.2578125,
           "text": "<image>"
         },
         {
@@ -35395,62 +20386,62 @@
         },
         {
           "id": 32000,
-          "logprob": -13.78125,
+          "logprob": -14.3984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5703125,
+          "logprob": -15.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9765625,
+          "logprob": -13.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4765625,
+          "logprob": -12.3515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.125,
+          "logprob": -13.46875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0859375,
+          "logprob": -11.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9296875,
+          "logprob": -12.2890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.90625,
+          "logprob": -14.3515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0625,
+          "logprob": -13.140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0234375,
+          "logprob": -13.0234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.6953125,
+          "logprob": -12.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.890625,
+          "logprob": -11.8125,
           "text": "<image>"
         },
         {
@@ -35458,6 +20449,31 @@
           "logprob": -12.125,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -11.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.640625,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -12.4375,
@@ -35465,67 +20481,257 @@
         },
         {
           "id": 32000,
-          "logprob": -12.375,
+          "logprob": -11.46875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.28125,
+          "logprob": -11.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4921875,
+          "logprob": -12.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1640625,
+          "logprob": -11.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1953125,
+          "logprob": -11.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.4140625,
+          "logprob": -12.4453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6171875,
+          "logprob": -11.34375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.921875,
+          "logprob": -12.1015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6328125,
+          "logprob": -11.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2421875,
+          "logprob": -14.6015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0234375,
+          "logprob": -12.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6953125,
+          "logprob": -11.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1484375,
+          "logprob": -11.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1328125,
           "text": "<image>"
         },
         {
@@ -35535,27 +20741,27 @@
         },
         {
           "id": 32000,
-          "logprob": -11.7265625,
+          "logprob": -11.71875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6796875,
+          "logprob": -11.6875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4921875,
+          "logprob": -12.484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.875,
+          "logprob": -11.8828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5390625,
+          "logprob": -12.53125,
           "text": "<image>"
         },
         {
@@ -35570,27 +20776,27 @@
         },
         {
           "id": 32000,
-          "logprob": -13.71875,
+          "logprob": -13.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.78125,
+          "logprob": -14.8046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.578125,
+          "logprob": -11.609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5859375,
+          "logprob": -13.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.390625,
+          "logprob": -12.421875,
           "text": "<image>"
         },
         {
@@ -35600,17 +20806,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.65625,
+          "logprob": -13.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5,
+          "logprob": -12.5078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.109375,
+          "logprob": -13.125,
           "text": "<image>"
         },
         {
@@ -35620,37 +20826,37 @@
         },
         {
           "id": 32000,
-          "logprob": -12.2578125,
+          "logprob": -12.2890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7421875,
+          "logprob": -12.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1640625,
+          "logprob": -13.15625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9375,
+          "logprob": -11.9453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.265625,
+          "logprob": -12.28125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1640625,
+          "logprob": -12.1328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.8671875,
+          "logprob": -13.8203125,
           "text": "<image>"
         },
         {
@@ -35660,57 +20866,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.046875,
+          "logprob": -13.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.984375,
+          "logprob": -15.6953125,
           "text": "<image>"
         },
         {
@@ -35720,12 +20881,47 @@
         },
         {
           "id": 32000,
-          "logprob": -11.3984375,
+          "logprob": -14.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.796875,
+          "logprob": -13.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.234375,
           "text": "<image>"
         },
         {
@@ -35735,37 +20931,47 @@
         },
         {
           "id": 32000,
-          "logprob": -11.4296875,
+          "logprob": -11.828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9921875,
+          "logprob": -11.453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.59375,
+          "logprob": -11.453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8359375,
+          "logprob": -11.984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7421875,
+          "logprob": -11.609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.125,
+          "logprob": -11.8046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.875,
+          "logprob": -11.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8515625,
           "text": "<image>"
         },
         {
@@ -35775,27 +20981,27 @@
         },
         {
           "id": 32000,
-          "logprob": -10.796875,
+          "logprob": -10.8046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4765625,
+          "logprob": -11.578125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1484375,
+          "logprob": -11.15625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8046875,
+          "logprob": -12.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3828125,
+          "logprob": -11.3671875,
           "text": "<image>"
         },
         {
@@ -35815,7 +21021,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.5546875,
+          "logprob": -12.5625,
           "text": "<image>"
         },
         {
@@ -35825,367 +21031,57 @@
         },
         {
           "id": 32000,
-          "logprob": -10.40625,
+          "logprob": -10.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.59375,
+          "logprob": -12.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.03125,
+          "logprob": -12.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.03125,
+          "logprob": -12.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.65625,
+          "logprob": -12.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6328125,
+          "logprob": -11.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5,
+          "logprob": -13.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.078125,
+          "logprob": -12.0703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0234375,
+          "logprob": -13.015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.171875,
+          "logprob": -14.1484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.4609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.015625,
+          "logprob": -12.3671875,
           "text": "<image>"
         },
         {
@@ -36195,42 +21091,112 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0859375,
+          "logprob": -12.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.140625,
+          "logprob": -13.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.65625,
+          "logprob": -13.953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.890625,
+          "logprob": -12.4765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.59375,
+          "logprob": -14.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4375,
+          "logprob": -12.6015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4375,
+          "logprob": -13.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.109375,
+          "logprob": -15.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0546875,
           "text": "<image>"
         },
         {
@@ -36240,17 +21206,257 @@
         },
         {
           "id": 32000,
-          "logprob": -11.1015625,
+          "logprob": -11.9609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.203125,
+          "logprob": -14.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9921875,
+          "logprob": -12.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.1484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.96875,
           "text": "<image>"
         },
         {
@@ -36260,32 +21466,32 @@
         },
         {
           "id": 32000,
-          "logprob": -11.40625,
+          "logprob": -11.4296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.53125,
+          "logprob": -12.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.109375,
+          "logprob": -12.1015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3203125,
+          "logprob": -11.3125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.125,
+          "logprob": -11.1171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7578125,
+          "logprob": -11.75,
           "text": "<image>"
         },
         {
@@ -36300,7 +21506,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.109375,
+          "logprob": -11.1015625,
           "text": "<image>"
         },
         {
@@ -36320,67 +21526,67 @@
         },
         {
           "id": 32000,
-          "logprob": -12.609375,
+          "logprob": -12.6015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.65625,
+          "logprob": -12.6796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7890625,
+          "logprob": -12.8359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.28125,
+          "logprob": -12.296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4140625,
+          "logprob": -13.3671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.96875,
+          "logprob": -12.9765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1171875,
+          "logprob": -12.109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5390625,
+          "logprob": -12.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.609375,
+          "logprob": -12.6015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6484375,
+          "logprob": -12.625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.09375,
+          "logprob": -12.109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9921875,
+          "logprob": -13.0,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4453125,
+          "logprob": -13.4296875,
           "text": "<image>"
         },
         {
@@ -36390,17 +21596,17 @@
         },
         {
           "id": 32000,
-          "logprob": -14.7109375,
+          "logprob": -14.71875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.625,
+          "logprob": -13.6015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.765625,
+          "logprob": -13.7578125,
           "text": "<image>"
         },
         {
@@ -36410,37 +21616,37 @@
         },
         {
           "id": 32000,
-          "logprob": -14.53125,
+          "logprob": -14.5390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.34375,
+          "logprob": -13.3515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.921875,
+          "logprob": -12.9453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.359375,
+          "logprob": -11.3515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.078125,
+          "logprob": -12.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4609375,
+          "logprob": -11.421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.640625,
+          "logprob": -11.625,
           "text": "<image>"
         },
         {
@@ -36455,62 +21661,62 @@
         },
         {
           "id": 32000,
-          "logprob": -11.953125,
+          "logprob": -11.9453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0859375,
+          "logprob": -12.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.421875,
+          "logprob": -11.4296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5859375,
+          "logprob": -11.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0859375,
+          "logprob": -12.0703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.25,
+          "logprob": -11.2421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6171875,
+          "logprob": -11.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3359375,
+          "logprob": -11.3125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.46875,
+          "logprob": -11.453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.265625,
+          "logprob": -12.28125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.21875,
+          "logprob": -12.2109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1875,
+          "logprob": -12.203125,
           "text": "<image>"
         },
         {
@@ -36525,47 +21731,47 @@
         },
         {
           "id": 32000,
-          "logprob": -11.2421875,
+          "logprob": -11.25,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3046875,
+          "logprob": -12.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.03125,
+          "logprob": -11.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.203125,
+          "logprob": -11.1953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.125,
+          "logprob": -11.1171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9296875,
+          "logprob": -11.9375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9765625,
+          "logprob": -10.984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.453125,
+          "logprob": -11.390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.859375,
+          "logprob": -12.8828125,
           "text": "<image>"
         },
         {
@@ -36590,47 +21796,47 @@
         },
         {
           "id": 32000,
-          "logprob": -12.1328125,
+          "logprob": -12.1171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5,
+          "logprob": -12.515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8671875,
+          "logprob": -11.890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1640625,
+          "logprob": -13.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.765625,
+          "logprob": -15.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.609375,
+          "logprob": -11.6015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.34375,
+          "logprob": -13.328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.671875,
+          "logprob": -13.6484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.9140625,
+          "logprob": -14.875,
           "text": "<image>"
         },
         {
@@ -36640,57 +21846,87 @@
         },
         {
           "id": 32000,
-          "logprob": -13.53125,
+          "logprob": -13.5234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.25,
+          "logprob": -13.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6171875,
+          "logprob": -12.609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.8046875,
+          "logprob": -13.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.375,
+          "logprob": -14.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.171875,
+          "logprob": -12.1484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.828125,
+          "logprob": -11.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.875,
+          "logprob": -11.890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.484375,
+          "logprob": -11.4921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5625,
+          "logprob": -12.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.640625,
+          "logprob": -11.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.03125,
           "text": "<image>"
         },
         {
@@ -36698,36 +21934,6 @@
           "logprob": -11.7265625,
           "text": "<image>"
         },
-        {
-          "id": 32000,
-          "logprob": -11.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.734375,
-          "text": "<image>"
-        },
         {
           "id": 32000,
           "logprob": -11.84375,
@@ -36735,12 +21941,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0390625,
+          "logprob": -11.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.2578125,
+          "logprob": -11.2421875,
           "text": "<image>"
         },
         {
@@ -36775,22 +21981,22 @@
         },
         {
           "id": 32000,
-          "logprob": -11.0390625,
+          "logprob": -11.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1953125,
+          "logprob": -12.203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.6640625,
+          "logprob": -13.609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6640625,
+          "logprob": -12.65625,
           "text": "<image>"
         },
         {
@@ -36815,27 +22021,47 @@
         },
         {
           "id": 32000,
-          "logprob": -11.9140625,
+          "logprob": -11.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.34375,
+          "logprob": -11.3359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.578125,
+          "logprob": -11.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.953125,
+          "logprob": -11.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.296875,
+          "logprob": -11.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.875,
           "text": "<image>"
         },
         {
@@ -36845,27 +22071,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9765625,
+          "logprob": -12.046875,
           "text": "<image>"
         },
         {
@@ -36875,7 +22081,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.734375,
+          "logprob": -13.7265625,
           "text": "<image>"
         },
         {
@@ -36885,7 +22091,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.8359375,
+          "logprob": -13.8203125,
           "text": "<image>"
         },
         {
@@ -36900,37 +22106,37 @@
         },
         {
           "id": 32000,
-          "logprob": -14.5625,
+          "logprob": -14.546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.0234375,
+          "logprob": -14.015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5703125,
+          "logprob": -12.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8125,
+          "logprob": -11.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.7109375,
+          "logprob": -10.6953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6015625,
+          "logprob": -11.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8671875,
+          "logprob": -12.0234375,
           "text": "<image>"
         },
         {
@@ -36945,32 +22151,32 @@
         },
         {
           "id": 32000,
-          "logprob": -14.3359375,
+          "logprob": -14.34375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.90625,
+          "logprob": -11.8984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.296875,
+          "logprob": -13.4375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.984375,
+          "logprob": -11.9765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0625,
+          "logprob": -11.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4296875,
+          "logprob": -11.421875,
           "text": "<image>"
         },
         {
@@ -36985,27 +22191,27 @@
         },
         {
           "id": 32000,
-          "logprob": -11.28125,
+          "logprob": -11.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5546875,
+          "logprob": -11.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.671875,
+          "logprob": -11.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1171875,
+          "logprob": -12.125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5,
+          "logprob": -11.484375,
           "text": "<image>"
         },
         {
@@ -37015,27 +22221,27 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8984375,
+          "logprob": -11.875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2890625,
+          "logprob": -12.3125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6484375,
+          "logprob": -11.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8984375,
+          "logprob": -12.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.734375,
+          "logprob": -11.7421875,
           "text": "<image>"
         },
         {
@@ -37045,7 +22251,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.40625,
+          "logprob": -11.3984375,
           "text": "<image>"
         },
         {
@@ -37055,7 +22261,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.3515625,
+          "logprob": -11.34375,
           "text": "<image>"
         },
         {
@@ -37070,37 +22276,37 @@
         },
         {
           "id": 32000,
-          "logprob": -11.03125,
+          "logprob": -10.9609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.375,
+          "logprob": -13.265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6640625,
+          "logprob": -12.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.96875,
+          "logprob": -11.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.671875,
+          "logprob": -14.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.390625,
+          "logprob": -12.3828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.2890625,
+          "logprob": -14.3515625,
           "text": "<image>"
         },
         {
@@ -37110,32 +22316,32 @@
         },
         {
           "id": 32000,
-          "logprob": -15.1796875,
+          "logprob": -15.125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6875,
+          "logprob": -12.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7421875,
+          "logprob": -12.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.6484375,
+          "logprob": -14.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.546875,
+          "logprob": -13.5234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5703125,
+          "logprob": -12.5859375,
           "text": "<image>"
         },
         {
@@ -37145,32 +22351,17 @@
         },
         {
           "id": 32000,
-          "logprob": -14.390625,
+          "logprob": -14.3828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.96875,
+          "logprob": -12.9453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7421875,
+          "logprob": -12.609375,
           "text": "<image>"
         },
         {
@@ -37180,32 +22371,47 @@
         },
         {
           "id": 32000,
-          "logprob": -12.453125,
+          "logprob": -12.953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.8046875,
+          "logprob": -11.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.0078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.7890625,
+          "logprob": -12.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.3828125,
+          "logprob": -13.8828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.828125,
+          "logprob": -12.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8359375,
           "text": "<image>"
         },
         {
@@ -37215,12 +22421,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.0,
+          "logprob": -13.0078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1171875,
+          "logprob": -12.140625,
           "text": "<image>"
         },
         {
@@ -37230,17 +22436,17 @@
         },
         {
           "id": 32000,
-          "logprob": -14.015625,
+          "logprob": -13.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6328125,
+          "logprob": -11.625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4765625,
+          "logprob": -11.484375,
           "text": "<image>"
         },
         {
@@ -37250,7 +22456,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.5859375,
+          "logprob": -11.5703125,
           "text": "<image>"
         },
         {
@@ -37265,7 +22471,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0390625,
+          "logprob": -12.03125,
           "text": "<image>"
         },
         {
@@ -37275,7 +22481,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.4609375,
+          "logprob": -11.46875,
           "text": "<image>"
         },
         {
@@ -37285,7 +22491,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0625,
+          "logprob": -12.078125,
           "text": "<image>"
         },
         {
@@ -37295,142 +22501,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
+          "logprob": -11.296875,
           "text": "<image>"
         },
         {
@@ -37440,7 +22511,142 @@
         },
         {
           "id": 32000,
-          "logprob": -11.34375,
+          "logprob": -11.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.21875,
           "text": "<image>"
         },
         {
@@ -37450,52 +22656,52 @@
         },
         {
           "id": 32000,
-          "logprob": -11.3671875,
+          "logprob": -11.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0,
+          "logprob": -12.984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.1875,
+          "logprob": -15.15625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0,
+          "logprob": -11.9765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.75,
+          "logprob": -12.765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.515625,
+          "logprob": -13.4921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5703125,
+          "logprob": -11.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.703125,
+          "logprob": -11.6953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3671875,
+          "logprob": -11.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4921875,
+          "logprob": -11.5,
           "text": "<image>"
         },
         {
@@ -37505,12 +22711,12 @@
         },
         {
           "id": 32000,
-          "logprob": -14.5546875,
+          "logprob": -14.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.390625,
+          "logprob": -13.4140625,
           "text": "<image>"
         },
         {
@@ -37520,7 +22726,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.7265625,
+          "logprob": -11.71875,
           "text": "<image>"
         },
         {
@@ -37535,7 +22741,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.2421875,
+          "logprob": -13.2578125,
           "text": "<image>"
         },
         {
@@ -37555,27 +22761,27 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8203125,
+          "logprob": -11.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9296875,
+          "logprob": -11.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.796875,
+          "logprob": -11.765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1875,
+          "logprob": -11.1953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.359375,
+          "logprob": -13.3359375,
           "text": "<image>"
         },
         {
@@ -37585,7 +22791,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.6796875,
+          "logprob": -12.703125,
           "text": "<image>"
         },
         {
@@ -37600,27 +22806,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.5625,
+          "logprob": -13.6171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6484375,
+          "logprob": -13.328125,
           "text": "<image>"
         },
         {
@@ -37628,6 +22819,21 @@
           "logprob": -12.828125,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -13.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.796875,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -12.78125,
@@ -37635,42 +22841,42 @@
         },
         {
           "id": 32000,
-          "logprob": -12.578125,
+          "logprob": -12.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7578125,
+          "logprob": -12.765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.484375,
+          "logprob": -11.46875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5390625,
+          "logprob": -12.484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6015625,
+          "logprob": -11.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.640625,
+          "logprob": -11.6171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.140625,
+          "logprob": -12.1328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.65625,
+          "logprob": -12.59375,
           "text": "<image>"
         },
         {
@@ -37680,17 +22886,17 @@
         },
         {
           "id": 32000,
-          "logprob": -14.265625,
+          "logprob": -14.25,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0078125,
+          "logprob": -12.0234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7578125,
+          "logprob": -11.7734375,
           "text": "<image>"
         },
         {
@@ -37705,52 +22911,52 @@
         },
         {
           "id": 32000,
-          "logprob": -11.625,
+          "logprob": -11.609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.53125,
+          "logprob": -13.5234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.046875,
+          "logprob": -12.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6796875,
+          "logprob": -12.6953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7109375,
+          "logprob": -11.703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.890625,
+          "logprob": -11.8828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8359375,
+          "logprob": -11.84375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7734375,
+          "logprob": -11.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3671875,
+          "logprob": -12.3515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.3125,
+          "logprob": -14.3046875,
           "text": "<image>"
         },
         {
@@ -37760,22 +22966,22 @@
         },
         {
           "id": 32000,
-          "logprob": -11.921875,
+          "logprob": -11.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3203125,
+          "logprob": -11.328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1640625,
+          "logprob": -14.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.2421875,
+          "logprob": -11.25,
           "text": "<image>"
         },
         {
@@ -37785,7 +22991,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.078125,
           "text": "<image>"
         },
         {
@@ -37795,12 +23001,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.2578125,
+          "logprob": -12.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.7265625,
+          "logprob": -13.71875,
           "text": "<image>"
         },
         {
@@ -37820,12 +23026,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.3828125,
+          "logprob": -13.390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5390625,
+          "logprob": -12.515625,
           "text": "<image>"
         },
         {
@@ -37835,22 +23041,22 @@
         },
         {
           "id": 32000,
-          "logprob": -14.15625,
+          "logprob": -14.1796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5703125,
+          "logprob": -11.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5546875,
+          "logprob": -11.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.65625,
+          "logprob": -11.671875,
           "text": "<image>"
         },
         {
@@ -37860,47 +23066,47 @@
         },
         {
           "id": 32000,
-          "logprob": -12.6484375,
+          "logprob": -12.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5390625,
+          "logprob": -12.515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7734375,
+          "logprob": -11.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0859375,
+          "logprob": -13.1328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9140625,
+          "logprob": -12.8984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.6015625,
+          "logprob": -14.609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5390625,
+          "logprob": -11.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8828125,
+          "logprob": -11.8515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.078125,
+          "logprob": -13.0625,
           "text": "<image>"
         },
         {
@@ -37910,17 +23116,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.15625,
+          "logprob": -12.1328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.03125,
+          "logprob": -12.0234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4453125,
+          "logprob": -12.453125,
           "text": "<image>"
         },
         {
@@ -37930,7 +23136,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.9375,
+          "logprob": -11.9296875,
           "text": "<image>"
         },
         {
@@ -37945,42 +23151,42 @@
         },
         {
           "id": 32000,
-          "logprob": -12.7890625,
+          "logprob": -12.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0390625,
+          "logprob": -13.0234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8203125,
+          "logprob": -12.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.578125,
+          "logprob": -12.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.7578125,
+          "logprob": -10.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9609375,
+          "logprob": -11.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.484375,
+          "logprob": -11.4921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.265625,
+          "logprob": -11.25,
           "text": "<image>"
         },
         {
@@ -37990,17 +23196,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.71875,
+          "logprob": -12.6953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7890625,
+          "logprob": -11.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5234375,
+          "logprob": -11.515625,
           "text": "<image>"
         },
         {
@@ -38010,7 +23216,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.46875,
+          "logprob": -12.4765625,
           "text": "<image>"
         },
         {
@@ -38025,17 +23231,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.65625,
+          "logprob": -11.6484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5234375,
+          "logprob": -11.515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7890625,
+          "logprob": -12.8125,
           "text": "<image>"
         },
         {
@@ -38045,17 +23251,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.28125,
+          "logprob": -13.2890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.109375,
+          "logprob": -11.09375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.34375,
+          "logprob": -12.3359375,
           "text": "<image>"
         },
         {
@@ -38065,27 +23271,27 @@
         },
         {
           "id": 32000,
-          "logprob": -13.0,
+          "logprob": -13.0078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.28125,
+          "logprob": -15.328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.8046875,
+          "logprob": -14.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0234375,
+          "logprob": -11.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.109375,
+          "logprob": -13.0703125,
           "text": "<image>"
         },
         {
@@ -38095,12 +23301,12 @@
         },
         {
           "id": 32000,
-          "logprob": -14.0078125,
+          "logprob": -14.0859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.7109375,
+          "logprob": -14.734375,
           "text": "<image>"
         },
         {
@@ -38115,127 +23321,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.9765625,
+          "logprob": -13.953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.3046875,
+          "logprob": -14.28125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.78125,
+          "logprob": -14.8828125,
           "text": "<image>"
         },
         {
@@ -38245,22 +23341,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.859375,
+          "logprob": -12.28125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.765625,
+          "logprob": -12.34375,
           "text": "<image>"
         },
         {
@@ -38275,117 +23361,87 @@
         },
         {
           "id": 32000,
-          "logprob": -11.71875,
+          "logprob": -12.4296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.25,
+          "logprob": -11.8984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.65625,
+          "logprob": -11.984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.84375,
+          "logprob": -11.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0390625,
+          "logprob": -11.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4921875,
+          "logprob": -12.1875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8828125,
+          "logprob": -11.6171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1015625,
+          "logprob": -12.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.0625,
+          "logprob": -12.4765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.09375,
+          "logprob": -12.953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2578125,
+          "logprob": -10.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.4453125,
+          "logprob": -13.1015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9921875,
+          "logprob": -12.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.140625,
+          "logprob": -11.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.828125,
+          "logprob": -11.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5546875,
+          "logprob": -11.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6015625,
+          "logprob": -11.765625,
           "text": "<image>"
         },
         {
@@ -38395,32 +23451,182 @@
         },
         {
           "id": 32000,
-          "logprob": -12.84375,
+          "logprob": -12.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.78125,
+          "logprob": -11.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1171875,
+          "logprob": -12.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8046875,
+          "logprob": -11.71875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5234375,
+          "logprob": -11.34375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4609375,
+          "logprob": -12.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4765625,
           "text": "<image>"
         },
         {
@@ -38440,22 +23646,22 @@
         },
         {
           "id": 32000,
-          "logprob": -12.953125,
+          "logprob": -12.9609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.859375,
+          "logprob": -12.8515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0390625,
+          "logprob": -12.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.28125,
+          "logprob": -13.234375,
           "text": "<image>"
         },
         {
@@ -38465,12 +23671,12 @@
         },
         {
           "id": 32000,
-          "logprob": -10.8203125,
+          "logprob": -10.8515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.6953125,
+          "logprob": -10.671875,
           "text": "<image>"
         },
         {
@@ -38480,7 +23686,7 @@
         },
         {
           "id": 32000,
-          "logprob": -10.9375,
+          "logprob": -10.9140625,
           "text": "<image>"
         },
         {
@@ -38500,17 +23706,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.3046875,
+          "logprob": -12.2890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9609375,
+          "logprob": -10.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6328125,
+          "logprob": -11.6015625,
           "text": "<image>"
         },
         {
@@ -38520,17 +23726,17 @@
         },
         {
           "id": 32000,
-          "logprob": -10.84375,
+          "logprob": -10.8515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.140625,
+          "logprob": -11.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.09375,
+          "logprob": -11.0859375,
           "text": "<image>"
         },
         {
@@ -38540,12 +23746,12 @@
         },
         {
           "id": 32000,
-          "logprob": -15.578125,
+          "logprob": -15.765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.875,
+          "logprob": -11.8515625,
           "text": "<image>"
         },
         {
@@ -38560,22 +23766,22 @@
         },
         {
           "id": 32000,
-          "logprob": -10.984375,
+          "logprob": -10.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.234375,
+          "logprob": -12.25,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8359375,
+          "logprob": -11.828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8515625,
+          "logprob": -11.84375,
           "text": "<image>"
         },
         {
@@ -38590,7 +23796,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.5625,
+          "logprob": -13.5703125,
           "text": "<image>"
         },
         {
@@ -38600,37 +23806,37 @@
         },
         {
           "id": 32000,
-          "logprob": -13.5859375,
+          "logprob": -13.6171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.0234375,
+          "logprob": -14.0078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1796875,
+          "logprob": -11.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1484375,
+          "logprob": -14.140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.234375,
+          "logprob": -12.25,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.734375,
+          "logprob": -11.7265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3515625,
+          "logprob": -12.3203125,
           "text": "<image>"
         },
         {
@@ -38640,17 +23846,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.90625,
+          "logprob": -11.8828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3359375,
+          "logprob": -12.3125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5703125,
+          "logprob": -11.5625,
           "text": "<image>"
         },
         {
@@ -38660,17 +23866,17 @@
         },
         {
           "id": 32000,
-          "logprob": -14.6875,
+          "logprob": -14.6796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0625,
+          "logprob": -11.046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5234375,
+          "logprob": -11.5,
           "text": "<image>"
         },
         {
@@ -38695,12 +23901,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.46875,
+          "logprob": -12.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.0859375,
           "text": "<image>"
         },
         {
@@ -38713,21 +23919,6 @@
           "logprob": -12.46875,
           "text": "<image>"
         },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6640625,
-          "text": "<image>"
-        },
         {
           "id": 32000,
           "logprob": -11.9765625,
@@ -38735,7 +23926,22 @@
         },
         {
           "id": 32000,
-          "logprob": -10.8671875,
+          "logprob": -11.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.875,
           "text": "<image>"
         },
         {
@@ -38755,12 +23961,12 @@
         },
         {
           "id": 32000,
-          "logprob": -10.796875,
+          "logprob": -10.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.59375,
+          "logprob": -11.6171875,
           "text": "<image>"
         },
         {
@@ -38790,22 +23996,22 @@
         },
         {
           "id": 32000,
-          "logprob": -11.390625,
+          "logprob": -11.3828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9375,
+          "logprob": -10.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.6796875,
+          "logprob": -10.703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9375,
+          "logprob": -11.90625,
           "text": "<image>"
         },
         {
@@ -38815,82 +24021,57 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8125,
+          "logprob": -11.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3671875,
+          "logprob": -11.4453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -9.625,
+          "logprob": -9.6328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.8515625,
+          "logprob": -10.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9453125,
+          "logprob": -13.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8203125,
+          "logprob": -12.890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3359375,
+          "logprob": -13.4765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.015625,
+          "logprob": -14.109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8671875,
+          "logprob": -11.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4453125,
+          "logprob": -13.484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.75,
+          "logprob": -13.5625,
           "text": "<image>"
         },
         {
@@ -38898,6 +24079,31 @@
           "logprob": -12.859375,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -10.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.890625,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -13.0625,
@@ -38905,7 +24111,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.21875,
+          "logprob": -11.2265625,
           "text": "<image>"
         },
         {
@@ -38915,7 +24121,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.296875,
+          "logprob": -12.2890625,
           "text": "<image>"
         },
         {
@@ -38925,12 +24131,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.8671875,
+          "logprob": -12.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.6953125,
+          "logprob": -15.671875,
           "text": "<image>"
         },
         {
@@ -38945,7 +24151,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.140625,
+          "logprob": -12.1328125,
           "text": "<image>"
         },
         {
@@ -38955,22 +24161,22 @@
         },
         {
           "id": 32000,
-          "logprob": -11.390625,
+          "logprob": -11.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.328125,
+          "logprob": -12.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.265625,
+          "logprob": -12.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9453125,
+          "logprob": -10.9296875,
           "text": "<image>"
         },
         {
@@ -38985,7 +24191,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.6015625,
+          "logprob": -11.59375,
           "text": "<image>"
         },
         {
@@ -38995,7 +24201,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.546875,
+          "logprob": -11.53125,
           "text": "<image>"
         },
         {
@@ -39010,7 +24216,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.6796875,
+          "logprob": -11.6875,
           "text": "<image>"
         },
         {
@@ -39025,27 +24231,27 @@
         },
         {
           "id": 32000,
-          "logprob": -12.125,
+          "logprob": -12.1171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5703125,
+          "logprob": -12.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.109375,
+          "logprob": -12.1015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0078125,
+          "logprob": -12.0,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.734375,
+          "logprob": -13.7265625,
           "text": "<image>"
         },
         {
@@ -39055,67 +24261,67 @@
         },
         {
           "id": 32000,
-          "logprob": -11.6484375,
+          "logprob": -11.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.34375,
+          "logprob": -12.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.140625,
+          "logprob": -12.1484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1796875,
+          "logprob": -14.21875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.8046875,
+          "logprob": -14.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -18.046875,
+          "logprob": -17.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.34375,
+          "logprob": -15.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.4453125,
+          "logprob": -14.4765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.625,
+          "logprob": -10.6015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0546875,
+          "logprob": -12.0703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.859375,
+          "logprob": -12.84375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2265625,
+          "logprob": -12.2421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2109375,
+          "logprob": -13.203125,
           "text": "<image>"
         },
         {
@@ -39125,37 +24331,37 @@
         },
         {
           "id": 32000,
-          "logprob": -10.9296875,
+          "logprob": -10.9375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9140625,
+          "logprob": -11.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.125,
+          "logprob": -12.1015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9609375,
+          "logprob": -10.9453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7734375,
+          "logprob": -11.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.828125,
+          "logprob": -11.8203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2890625,
+          "logprob": -12.296875,
           "text": "<image>"
         },
         {
@@ -39165,27 +24371,27 @@
         },
         {
           "id": 32000,
-          "logprob": -11.53125,
+          "logprob": -11.5390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0234375,
+          "logprob": -12.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.2421875,
+          "logprob": -15.2265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1171875,
+          "logprob": -12.125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1171875,
+          "logprob": -12.125,
           "text": "<image>"
         },
         {
@@ -39195,7 +24401,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.671875,
+          "logprob": -11.6796875,
           "text": "<image>"
         },
         {
@@ -39215,32 +24421,32 @@
         },
         {
           "id": 32000,
-          "logprob": -12.734375,
+          "logprob": -12.703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.609375,
+          "logprob": -11.6328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5234375,
+          "logprob": -11.515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5078125,
+          "logprob": -11.546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2109375,
+          "logprob": -12.21875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.421875,
+          "logprob": -11.4140625,
           "text": "<image>"
         },
         {
@@ -39250,137 +24456,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.96875,
+          "logprob": -11.9765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.46875,
+          "logprob": -12.9609375,
           "text": "<image>"
         },
         {
@@ -39388,6 +24469,131 @@
           "logprob": -12.578125,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -11.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5078125,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -11.6171875,
@@ -39395,32 +24601,32 @@
         },
         {
           "id": 32000,
-          "logprob": -13.8125,
+          "logprob": -13.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1640625,
+          "logprob": -11.203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.765625,
+          "logprob": -11.78125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.5,
+          "logprob": -16.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3046875,
+          "logprob": -12.296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1171875,
+          "logprob": -14.1484375,
           "text": "<image>"
         },
         {
@@ -39430,12 +24636,47 @@
         },
         {
           "id": 32000,
-          "logprob": -12.5,
+          "logprob": -12.4921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.6015625,
+          "logprob": -10.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1953125,
           "text": "<image>"
         },
         {
@@ -39445,52 +24686,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.625,
+          "logprob": -11.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3046875,
+          "logprob": -12.6484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1171875,
+          "logprob": -11.109375,
           "text": "<image>"
         },
         {
@@ -39500,7 +24706,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.921875,
+          "logprob": -13.90625,
           "text": "<image>"
         },
         {
@@ -39515,12 +24721,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.3671875,
+          "logprob": -13.375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.7109375,
+          "logprob": -14.703125,
           "text": "<image>"
         },
         {
@@ -39530,7 +24736,7 @@
         },
         {
           "id": 32000,
-          "logprob": -14.859375,
+          "logprob": -14.890625,
           "text": "<image>"
         },
         {
@@ -39545,37 +24751,37 @@
         },
         {
           "id": 32000,
-          "logprob": -11.9296875,
+          "logprob": -11.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3828125,
+          "logprob": -13.3984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.046875,
+          "logprob": -13.9453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1484375,
+          "logprob": -13.1875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.265625,
+          "logprob": -11.28125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.125,
+          "logprob": -15.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.046875,
+          "logprob": -15.0234375,
           "text": "<image>"
         },
         {
@@ -39590,92 +24796,22 @@
         },
         {
           "id": 32000,
-          "logprob": -14.09375,
+          "logprob": -14.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.890625,
+          "logprob": -13.8984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.3203125,
+          "logprob": -15.3125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9609375,
+          "logprob": -12.8125,
           "text": "<image>"
         },
         {
@@ -39685,22 +24821,92 @@
         },
         {
           "id": 32000,
-          "logprob": -12.3203125,
+          "logprob": -13.046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.59375,
+          "logprob": -10.625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.171875,
+          "logprob": -11.25,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5,
+          "logprob": -12.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4921875,
           "text": "<image>"
         },
         {
@@ -39730,17 +24936,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.78125,
+          "logprob": -12.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3125,
+          "logprob": -11.3046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1171875,
+          "logprob": -11.109375,
           "text": "<image>"
         },
         {
@@ -39750,27 +24956,27 @@
         },
         {
           "id": 32000,
-          "logprob": -13.0859375,
+          "logprob": -13.0234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.921875,
+          "logprob": -10.9296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.515625,
+          "logprob": -13.5078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4921875,
+          "logprob": -11.5,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8203125,
+          "logprob": -11.8359375,
           "text": "<image>"
         },
         {
@@ -39780,42 +24986,42 @@
         },
         {
           "id": 32000,
-          "logprob": -14.0078125,
+          "logprob": -14.0234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.21875,
+          "logprob": -15.2578125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -17.84375,
+          "logprob": -17.953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.046875,
+          "logprob": -12.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.296875,
+          "logprob": -12.28125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7109375,
+          "logprob": -11.6875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1015625,
+          "logprob": -12.1171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.515625,
+          "logprob": -11.5546875,
           "text": "<image>"
         },
         {
@@ -39825,17 +25031,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.640625,
+          "logprob": -12.6171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4609375,
+          "logprob": -13.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.84375,
+          "logprob": -14.828125,
           "text": "<image>"
         },
         {
@@ -39845,22 +25051,22 @@
         },
         {
           "id": 32000,
-          "logprob": -11.6171875,
+          "logprob": -11.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.765625,
+          "logprob": -12.7421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5390625,
+          "logprob": -13.484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4453125,
+          "logprob": -11.4609375,
           "text": "<image>"
         },
         {
@@ -39870,37 +25076,37 @@
         },
         {
           "id": 32000,
-          "logprob": -11.96875,
+          "logprob": -11.984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4765625,
+          "logprob": -12.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.890625,
+          "logprob": -15.9375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.953125,
+          "logprob": -11.984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.40625,
+          "logprob": -11.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0,
+          "logprob": -12.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1875,
+          "logprob": -14.3046875,
           "text": "<image>"
         },
         {
@@ -39910,7 +25116,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.125,
+          "logprob": -12.1171875,
           "text": "<image>"
         },
         {
@@ -39920,22 +25126,22 @@
         },
         {
           "id": 32000,
-          "logprob": -13.140625,
+          "logprob": -13.1484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1484375,
+          "logprob": -12.1875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.375,
+          "logprob": -13.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.9453125,
+          "logprob": -13.9921875,
           "text": "<image>"
         },
         {
@@ -39945,12 +25151,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0859375,
+          "logprob": -12.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.734375,
+          "logprob": -10.7421875,
           "text": "<image>"
         },
         {
@@ -39960,12 +25166,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.65625,
+          "logprob": -12.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.953125,
+          "logprob": -11.9765625,
           "text": "<image>"
         },
         {
@@ -39975,12 +25181,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.5,
+          "logprob": -12.4765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8984375,
+          "logprob": -11.890625,
           "text": "<image>"
         },
         {
@@ -40000,22 +25206,22 @@
         },
         {
           "id": 32000,
-          "logprob": -12.4375,
+          "logprob": -12.4609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5703125,
+          "logprob": -12.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.828125,
+          "logprob": -12.8203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.375,
+          "logprob": -13.359375,
           "text": "<image>"
         },
         {
@@ -40035,7 +25241,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.2421875,
+          "logprob": -11.234375,
           "text": "<image>"
         },
         {
@@ -40045,27 +25251,27 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.328125,
+          "logprob": -15.3046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.890625,
+          "logprob": -12.8828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4453125,
+          "logprob": -13.453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8203125,
+          "logprob": -11.8125,
           "text": "<image>"
         },
         {
@@ -40075,17 +25281,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.96875,
+          "logprob": -12.9765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9921875,
+          "logprob": -11.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.421875,
+          "logprob": -13.4296875,
           "text": "<image>"
         },
         {
@@ -40095,12 +25301,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.265625,
+          "logprob": -13.28125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.265625,
+          "logprob": -13.2578125,
           "text": "<image>"
         },
         {
@@ -40115,12 +25321,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.15625,
+          "logprob": -12.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.375,
+          "logprob": -10.3828125,
           "text": "<image>"
         },
         {
@@ -40130,7 +25336,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.2109375,
+          "logprob": -13.203125,
           "text": "<image>"
         },
         {
@@ -40140,7 +25346,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.15625,
+          "logprob": -12.1328125,
           "text": "<image>"
         },
         {
@@ -40150,12 +25356,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.171875,
+          "logprob": -13.1796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6328125,
+          "logprob": -11.65625,
           "text": "<image>"
         },
         {
@@ -40165,12 +25371,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.90625,
+          "logprob": -11.9375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.765625,
+          "logprob": -11.7734375,
           "text": "<image>"
         },
         {
@@ -40185,7 +25391,7 @@
         },
         {
           "id": 32000,
-          "logprob": -10.8984375,
+          "logprob": -10.90625,
           "text": "<image>"
         },
         {
@@ -40195,17 +25401,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.40625,
+          "logprob": -12.3828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5390625,
+          "logprob": -11.5234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.59375,
+          "logprob": -16.578125,
           "text": "<image>"
         },
         {
@@ -40215,7 +25421,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.9140625,
+          "logprob": -11.9453125,
           "text": "<image>"
         },
         {
@@ -40230,22 +25436,22 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8984375,
+          "logprob": -11.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0703125,
+          "logprob": -13.0625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.03125,
+          "logprob": -12.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.921875,
+          "logprob": -11.9296875,
           "text": "<image>"
         },
         {
@@ -40255,12 +25461,12 @@
         },
         {
           "id": 32000,
-          "logprob": -10.5390625,
+          "logprob": -10.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.453125,
+          "logprob": -11.4375,
           "text": "<image>"
         },
         {
@@ -40270,22 +25476,22 @@
         },
         {
           "id": 32000,
-          "logprob": -13.0859375,
+          "logprob": -13.09375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1796875,
+          "logprob": -12.15625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.8203125,
+          "logprob": -15.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.984375,
+          "logprob": -10.9765625,
           "text": "<image>"
         },
         {
@@ -40295,42 +25501,42 @@
         },
         {
           "id": 32000,
-          "logprob": -10.109375,
+          "logprob": -10.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8671875,
+          "logprob": -11.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1328125,
+          "logprob": -13.125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.640625,
+          "logprob": -12.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.015625,
+          "logprob": -16.046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5078125,
+          "logprob": -11.515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.7265625,
+          "logprob": -13.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.703125,
+          "logprob": -14.671875,
           "text": "<image>"
         },
         {
@@ -40340,32 +25546,32 @@
         },
         {
           "id": 32000,
-          "logprob": -12.2421875,
+          "logprob": -12.2109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.6640625,
+          "logprob": -10.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.6640625,
+          "logprob": -15.6484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9140625,
+          "logprob": -10.9296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3203125,
+          "logprob": -11.328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.046875,
+          "logprob": -11.0546875,
           "text": "<image>"
         },
         {
@@ -40375,7 +25581,7 @@
         },
         {
           "id": 32000,
-          "logprob": -10.953125,
+          "logprob": -10.96875,
           "text": "<image>"
         },
         {
@@ -40390,337 +25596,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2578125,
+          "logprob": -12.359375,
           "text": "<image>"
         },
         {
@@ -40730,22 +25606,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.84375,
+          "logprob": -14.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
+          "logprob": -11.4921875,
           "text": "<image>"
         },
         {
@@ -40755,87 +25621,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.125,
+          "logprob": -12.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.390625,
+          "logprob": -12.640625,
           "text": "<image>"
         },
         {
@@ -40845,87 +25636,297 @@
         },
         {
           "id": 32000,
-          "logprob": -12.5078125,
+          "logprob": -12.4375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.09375,
+          "logprob": -11.8828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.8515625,
+          "logprob": -12.15625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8203125,
+          "logprob": -11.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.734375,
+          "logprob": -13.0,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2578125,
+          "logprob": -12.0703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.28125,
+          "logprob": -11.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.859375,
+          "logprob": -13.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.703125,
+          "logprob": -12.1484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5703125,
+          "logprob": -13.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.59375,
+          "logprob": -12.203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.890625,
+          "logprob": -11.9375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3359375,
+          "logprob": -12.1796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0625,
+          "logprob": -10.7421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.53125,
+          "logprob": -10.8203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7109375,
+          "logprob": -12.375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.7578125,
+          "logprob": -11.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.25,
           "text": "<image>"
         },
         {
@@ -40935,17 +25936,47 @@
         },
         {
           "id": 32000,
-          "logprob": -10.75,
+          "logprob": -11.8515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.2265625,
+          "logprob": -13.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9296875,
+          "logprob": -10.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1328125,
           "text": "<image>"
         },
         {
@@ -40955,7 +25986,12 @@
         },
         {
           "id": 32000,
-          "logprob": -10.5390625,
+          "logprob": -14.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0546875,
           "text": "<image>"
         },
         {
@@ -40963,6 +25999,56 @@
           "logprob": -11.6796875,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -11.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.234375,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -12.5390625,
@@ -40970,7 +26056,127 @@
         },
         {
           "id": 32000,
-          "logprob": -16.953125,
+          "logprob": -15.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.0,
           "text": "<image>"
         },
         {
@@ -40980,12 +26186,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.3359375,
+          "logprob": -11.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.671875,
+          "logprob": -11.6796875,
           "text": "<image>"
         },
         {
@@ -41000,12 +26206,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8671875,
+          "logprob": -11.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.484375,
+          "logprob": -11.4765625,
           "text": "<image>"
         },
         {
@@ -41015,12 +26221,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.6640625,
+          "logprob": -12.6875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4296875,
+          "logprob": -13.421875,
           "text": "<image>"
         },
         {
@@ -41035,7 +26241,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.3046875,
+          "logprob": -12.3125,
           "text": "<image>"
         },
         {
@@ -41045,17 +26251,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.34375,
+          "logprob": -13.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0703125,
+          "logprob": -13.0625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.234375,
+          "logprob": -11.2421875,
           "text": "<image>"
         },
         {
@@ -41065,37 +26271,37 @@
         },
         {
           "id": 32000,
-          "logprob": -13.4921875,
+          "logprob": -13.5,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.7578125,
+          "logprob": -10.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.53125,
+          "logprob": -13.5390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6953125,
+          "logprob": -12.703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.203125,
+          "logprob": -13.1953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -17.078125,
+          "logprob": -17.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.25,
+          "logprob": -15.0234375,
           "text": "<image>"
         },
         {
@@ -41105,17 +26311,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.140625,
+          "logprob": -12.125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5234375,
+          "logprob": -11.515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.0625,
+          "logprob": -16.125,
           "text": "<image>"
         },
         {
@@ -41125,17 +26331,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.6015625,
+          "logprob": -11.609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.5859375,
+          "logprob": -14.6015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.890625,
+          "logprob": -11.8984375,
           "text": "<image>"
         },
         {
@@ -41150,22 +26356,22 @@
         },
         {
           "id": 32000,
-          "logprob": -11.7734375,
+          "logprob": -11.78125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9921875,
+          "logprob": -13.0,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9375,
+          "logprob": -12.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9921875,
+          "logprob": -12.0,
           "text": "<image>"
         },
         {
@@ -41175,7 +26381,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.7578125,
+          "logprob": -11.7734375,
           "text": "<image>"
         },
         {
@@ -41185,7 +26391,7 @@
         },
         {
           "id": 32000,
-          "logprob": -14.328125,
+          "logprob": -14.3203125,
           "text": "<image>"
         },
         {
@@ -41195,22 +26401,22 @@
         },
         {
           "id": 32000,
-          "logprob": -13.84375,
+          "logprob": -13.875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.2109375,
+          "logprob": -14.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.390625,
+          "logprob": -12.375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.953125,
+          "logprob": -13.9375,
           "text": "<image>"
         },
         {
@@ -41220,27 +26426,27 @@
         },
         {
           "id": 32000,
-          "logprob": -14.6796875,
+          "logprob": -14.609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.84375,
+          "logprob": -11.8359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.65625,
+          "logprob": -10.6484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0078125,
+          "logprob": -12.0,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0859375,
+          "logprob": -11.0703125,
           "text": "<image>"
         },
         {
@@ -41260,7 +26466,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.6875,
+          "logprob": -12.6640625,
           "text": "<image>"
         },
         {
@@ -41275,22 +26481,22 @@
         },
         {
           "id": 32000,
-          "logprob": -16.109375,
+          "logprob": -16.09375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.984375,
+          "logprob": -11.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.40625,
+          "logprob": -13.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.375,
+          "logprob": -13.453125,
           "text": "<image>"
         },
         {
@@ -41300,57 +26506,57 @@
         },
         {
           "id": 32000,
-          "logprob": -12.921875,
+          "logprob": -12.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.859375,
+          "logprob": -14.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3671875,
+          "logprob": -13.375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0546875,
+          "logprob": -12.0703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.21875,
+          "logprob": -15.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6953125,
+          "logprob": -11.6875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.5234375,
+          "logprob": -10.5390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.8984375,
+          "logprob": -14.703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.8515625,
+          "logprob": -13.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6796875,
+          "logprob": -11.6875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -17.4375,
+          "logprob": -17.453125,
           "text": "<image>"
         },
         {
@@ -41365,7 +26571,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8125,
+          "logprob": -11.7890625,
           "text": "<image>"
         },
         {
@@ -41380,12 +26586,12 @@
         },
         {
           "id": 32000,
-          "logprob": -14.2421875,
+          "logprob": -14.265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.7265625,
+          "logprob": -13.6953125,
           "text": "<image>"
         },
         {
@@ -41400,57 +26606,57 @@
         },
         {
           "id": 32000,
-          "logprob": -12.84375,
+          "logprob": -12.890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9140625,
+          "logprob": -11.9375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.7421875,
+          "logprob": -14.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.984375,
+          "logprob": -11.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.71875,
+          "logprob": -14.7265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.578125,
+          "logprob": -11.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.015625,
+          "logprob": -13.0,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.796875,
+          "logprob": -12.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0078125,
+          "logprob": -11.015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.46875,
+          "logprob": -16.421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9296875,
+          "logprob": -11.921875,
           "text": "<image>"
         },
         {
@@ -41460,7 +26666,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.8515625,
+          "logprob": -12.859375,
           "text": "<image>"
         },
         {
@@ -41475,12 +26681,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.140625,
+          "logprob": -12.1328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.359375,
+          "logprob": -13.3515625,
           "text": "<image>"
         },
         {
@@ -41495,27 +26701,27 @@
         },
         {
           "id": 32000,
-          "logprob": -11.578125,
+          "logprob": -11.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8203125,
+          "logprob": -12.84375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.984375,
+          "logprob": -11.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5078125,
+          "logprob": -12.515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.625,
+          "logprob": -12.6328125,
           "text": "<image>"
         },
         {
@@ -41530,7 +26736,7 @@
         },
         {
           "id": 32000,
-          "logprob": -14.0546875,
+          "logprob": -14.046875,
           "text": "<image>"
         },
         {
@@ -41540,7 +26746,7 @@
         },
         {
           "id": 32000,
-          "logprob": -17.21875,
+          "logprob": -17.203125,
           "text": "<image>"
         },
         {
@@ -41550,17 +26756,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.3671875,
+          "logprob": -13.375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.6171875,
+          "logprob": -14.609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9609375,
+          "logprob": -12.953125,
           "text": "<image>"
         },
         {
@@ -41570,12 +26776,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.7265625,
+          "logprob": -11.71875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.03125,
+          "logprob": -15.0234375,
           "text": "<image>"
         },
         {
@@ -41585,7 +26791,7 @@
         },
         {
           "id": 32000,
-          "logprob": -15.890625,
+          "logprob": -15.9296875,
           "text": "<image>"
         },
         {
@@ -41610,22 +26816,22 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0234375,
+          "logprob": -12.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7578125,
+          "logprob": -12.75,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.7578125,
+          "logprob": -14.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5,
+          "logprob": -11.484375,
           "text": "<image>"
         },
         {
@@ -41635,7 +26841,7 @@
         },
         {
           "id": 32000,
-          "logprob": -10.7265625,
+          "logprob": -10.734375,
           "text": "<image>"
         },
         {
@@ -41645,22 +26851,22 @@
         },
         {
           "id": 32000,
-          "logprob": -11.4140625,
+          "logprob": -11.421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9453125,
+          "logprob": -12.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1171875,
+          "logprob": -13.0625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0390625,
+          "logprob": -12.03125,
           "text": "<image>"
         },
         {
@@ -41670,7 +26876,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.234375,
+          "logprob": -12.2109375,
           "text": "<image>"
         },
         {
@@ -41685,7 +26891,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.0390625,
+          "logprob": -13.09375,
           "text": "<image>"
         },
         {
@@ -41695,7 +26901,7 @@
         },
         {
           "id": 32000,
-          "logprob": -15.265625,
+          "logprob": -15.2578125,
           "text": "<image>"
         },
         {
@@ -41710,7 +26916,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.453125,
+          "logprob": -11.4609375,
           "text": "<image>"
         },
         {
@@ -41725,12 +26931,12 @@
         },
         {
           "id": 32000,
-          "logprob": -10.6796875,
+          "logprob": -10.6875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.0546875,
           "text": "<image>"
         },
         {
@@ -41740,7 +26946,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.421875,
+          "logprob": -12.4140625,
           "text": "<image>"
         },
         {
@@ -41750,27 +26956,27 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8046875,
+          "logprob": -11.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.3828125,
+          "logprob": -14.4609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.734375,
+          "logprob": -11.7109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.890625,
+          "logprob": -12.828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.859375,
+          "logprob": -12.8359375,
           "text": "<image>"
         },
         {
@@ -41780,27 +26986,27 @@
         },
         {
           "id": 32000,
-          "logprob": -13.5625,
+          "logprob": -13.5390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.25,
+          "logprob": -16.21875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.8125,
+          "logprob": -16.84375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.921875,
+          "logprob": -11.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.375,
+          "logprob": -14.359375,
           "text": "<image>"
         },
         {
@@ -41810,12 +27016,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.671875,
+          "logprob": -13.6953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9921875,
+          "logprob": -12.984375,
           "text": "<image>"
         },
         {
@@ -41830,57 +27036,37 @@
         },
         {
           "id": 32000,
-          "logprob": -17.5625,
+          "logprob": -17.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.625,
+          "logprob": -12.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.0625,
+          "logprob": -15.046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.75,
+          "logprob": -16.71875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.84375,
+          "logprob": -11.8359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.328125,
+          "logprob": -13.3515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
+          "logprob": -12.2734375,
           "text": "<image>"
         },
         {
@@ -41890,17 +27076,37 @@
         },
         {
           "id": 32000,
-          "logprob": -13.109375,
+          "logprob": -11.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.625,
+          "logprob": -12.46875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.890625,
+          "logprob": -11.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.921875,
           "text": "<image>"
         },
         {
@@ -41910,7 +27116,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.15625,
+          "logprob": -13.21875,
           "text": "<image>"
         },
         {
@@ -41930,7 +27136,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.5390625,
+          "logprob": -12.5625,
           "text": "<image>"
         },
         {
@@ -41940,12 +27146,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.9609375,
+          "logprob": -11.9453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.828125,
+          "logprob": -11.8359375,
           "text": "<image>"
         },
         {
@@ -41955,22 +27161,22 @@
         },
         {
           "id": 32000,
-          "logprob": -12.8125,
+          "logprob": -12.875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.96875,
+          "logprob": -11.9609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.109375,
+          "logprob": -13.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6328125,
+          "logprob": -12.625,
           "text": "<image>"
         },
         {
@@ -41990,12 +27196,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.109375,
+          "logprob": -12.0859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.265625,
+          "logprob": -14.25,
           "text": "<image>"
         },
         {
@@ -42005,127 +27211,17 @@
         },
         {
           "id": 32000,
-          "logprob": -16.640625,
+          "logprob": -16.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.828125,
+          "logprob": -12.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -19.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5859375,
+          "logprob": -12.7890625,
           "text": "<image>"
         },
         {
@@ -42135,852 +27231,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.3046875,
+          "logprob": -12.3125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5234375,
+          "logprob": -12.3203125,
           "text": "<image>"
         },
         {
@@ -42990,107 +27246,137 @@
         },
         {
           "id": 32000,
-          "logprob": -11.6875,
+          "logprob": -14.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.765625,
+          "logprob": -14.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4140625,
+          "logprob": -12.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -18.703125,
+          "logprob": -14.546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.6484375,
+          "logprob": -19.4375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.2109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.703125,
+          "logprob": -13.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.0859375,
+          "logprob": -11.8515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.0703125,
+          "logprob": -11.4453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.5234375,
+          "logprob": -12.2265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3671875,
+          "logprob": -12.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0390625,
+          "logprob": -12.3359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.640625,
+          "logprob": -17.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8359375,
+          "logprob": -17.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1015625,
+          "logprob": -11.515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3515625,
+          "logprob": -12.0078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.578125,
+          "logprob": -13.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.25,
+          "logprob": -12.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.15625,
+          "logprob": -12.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7265625,
+          "logprob": -12.328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.90625,
+          "logprob": -13.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.5,
           "text": "<image>"
         },
         {
@@ -43105,17 +27391,32 @@
         },
         {
           "id": 32000,
-          "logprob": -10.5625,
+          "logprob": -14.875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.171875,
+          "logprob": -11.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2265625,
+          "logprob": -11.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8984375,
           "text": "<image>"
         },
         {
@@ -43125,137 +27426,37 @@
         },
         {
           "id": 32000,
-          "logprob": -11.7578125,
+          "logprob": -11.328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.875,
+          "logprob": -11.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4453125,
+          "logprob": -11.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1796875,
+          "logprob": -15.046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9453125,
+          "logprob": -12.2109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8203125,
+          "logprob": -12.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4765625,
+          "logprob": -16.875,
           "text": "<image>"
         },
         {
@@ -43265,22 +27466,227 @@
         },
         {
           "id": 32000,
-          "logprob": -15.484375,
+          "logprob": -14.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.6328125,
+          "logprob": -11.9609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.6015625,
+          "logprob": -12.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.75,
+          "logprob": -11.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.890625,
           "text": "<image>"
         },
         {
@@ -43290,7 +27696,807 @@
         },
         {
           "id": 32000,
-          "logprob": -13.3125,
+          "logprob": -13.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -18.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.0859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2578125,
           "text": "<image>"
         },
         {
@@ -43300,12 +28506,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.484375,
+          "logprob": -13.5390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2734375,
+          "logprob": -13.265625,
           "text": "<image>"
         },
         {
@@ -43320,17 +28526,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.6796875,
+          "logprob": -11.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.53125,
+          "logprob": -16.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1484375,
+          "logprob": -14.171875,
           "text": "<image>"
         },
         {
@@ -43340,17 +28546,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8125,
+          "logprob": -11.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.03125,
+          "logprob": -15.1171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.484375,
+          "logprob": -15.4453125,
           "text": "<image>"
         },
         {
@@ -43365,37 +28571,37 @@
         },
         {
           "id": 32000,
-          "logprob": -10.6953125,
+          "logprob": -10.703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1640625,
+          "logprob": -11.15625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.84375,
+          "logprob": -12.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5625,
+          "logprob": -12.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7421875,
+          "logprob": -12.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -18.0,
+          "logprob": -17.953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5859375,
+          "logprob": -11.59375,
           "text": "<image>"
         },
         {
@@ -43410,12 +28616,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.6796875,
+          "logprob": -11.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.71875,
+          "logprob": -12.8515625,
           "text": "<image>"
         },
         {
@@ -43425,27 +28631,27 @@
         },
         {
           "id": 32000,
-          "logprob": -12.1875,
+          "logprob": -12.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7578125,
+          "logprob": -12.7265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.125,
+          "logprob": -15.140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.9140625,
+          "logprob": -14.9375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.546875,
+          "logprob": -15.6875,
           "text": "<image>"
         },
         {
@@ -43455,22 +28661,22 @@
         },
         {
           "id": 32000,
-          "logprob": -14.109375,
+          "logprob": -14.125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.234375,
+          "logprob": -13.2265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.15625,
+          "logprob": -13.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.734375,
+          "logprob": -12.703125,
           "text": "<image>"
         },
         {
@@ -43480,7 +28686,7 @@
         },
         {
           "id": 32000,
-          "logprob": -14.7734375,
+          "logprob": -14.765625,
           "text": "<image>"
         },
         {
@@ -43490,12 +28696,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.7421875,
+          "logprob": -11.71875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.921875,
+          "logprob": -11.9296875,
           "text": "<image>"
         },
         {
@@ -43505,17 +28711,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.6015625,
+          "logprob": -13.6328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4375,
+          "logprob": -11.4296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5,
+          "logprob": -13.5078125,
           "text": "<image>"
         },
         {
@@ -43530,27 +28736,27 @@
         },
         {
           "id": 32000,
-          "logprob": -12.1328125,
+          "logprob": -12.15625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.90625,
+          "logprob": -13.1171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.578125,
+          "logprob": -12.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3359375,
+          "logprob": -13.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7265625,
+          "logprob": -11.734375,
           "text": "<image>"
         },
         {
@@ -43558,491 +28764,11 @@
           "logprob": -10.953125,
           "text": "<image>"
         },
-        {
-          "id": 32000,
-          "logprob": -12.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -19.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.796875,
-          "text": "<image>"
-        },
         {
           "id": 32000,
           "logprob": -12.75,
           "text": "<image>"
         },
-        {
-          "id": 32000,
-          "logprob": -14.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.1953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.515625,
-          "text": "<image>"
-        },
         {
           "id": 32000,
           "logprob": -12.7734375,
@@ -44050,7 +28776,22 @@
         },
         {
           "id": 32000,
-          "logprob": -12.4609375,
+          "logprob": -13.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5390625,
           "text": "<image>"
         },
         {
@@ -44060,87 +28801,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.171875,
+          "logprob": -19.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8828125,
+          "logprob": -12.4765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8828125,
+          "logprob": -11.609375,
           "text": "<image>"
         },
         {
@@ -44153,6 +28824,541 @@
           "logprob": -13.3203125,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -12.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.359375,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -13.2109375,
@@ -44160,7 +29366,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.9765625,
+          "logprob": -11.984375,
           "text": "<image>"
         },
         {
@@ -44170,12 +29376,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0078125,
+          "logprob": -12.015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0390625,
+          "logprob": -11.03125,
           "text": "<image>"
         },
         {
@@ -44190,17 +29396,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.8203125,
+          "logprob": -13.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5078125,
+          "logprob": -13.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.734375,
+          "logprob": -11.7265625,
           "text": "<image>"
         },
         {
@@ -44210,22 +29416,22 @@
         },
         {
           "id": 32000,
-          "logprob": -17.3125,
+          "logprob": -16.84375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5234375,
+          "logprob": -12.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -17.625,
+          "logprob": -17.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9296875,
+          "logprob": -11.9453125,
           "text": "<image>"
         },
         {
@@ -44235,17 +29441,17 @@
         },
         {
           "id": 32000,
-          "logprob": -15.9140625,
+          "logprob": -16.0,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.65625,
+          "logprob": -16.625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5,
+          "logprob": -12.4921875,
           "text": "<image>"
         },
         {
@@ -44255,17 +29461,17 @@
         },
         {
           "id": 368,
-          "logprob": -0.19726562,
+          "logprob": -0.19604492,
           "text": "you"
         },
         {
           "id": 1912,
-          "logprob": -1.4990234,
+          "logprob": -1.5058594,
           "text": "tell"
         },
         {
           "id": 528,
-          "logprob": -0.31152344,
+          "logprob": -0.31030273,
           "text": "me"
         },
         {
@@ -44280,22 +29486,22 @@
         },
         {
           "id": 2485,
-          "logprob": -0.9941406,
+          "logprob": -0.9975586,
           "text": "short"
         },
         {
           "id": 2838,
-          "logprob": -0.46118164,
+          "logprob": -0.4633789,
           "text": "story"
         },
         {
           "id": 2818,
-          "logprob": -3.3183594,
+          "logprob": -3.3144531,
           "text": "based"
         },
         {
           "id": 356,
-          "logprob": -0.029129028,
+          "logprob": -0.029037476,
           "text": "on"
         },
         {
@@ -44305,12 +29511,12 @@
         },
         {
           "id": 3469,
-          "logprob": -0.29052734,
+          "logprob": -0.2890625,
           "text": "image"
         },
         {
           "id": 28804,
-          "logprob": -0.43188477,
+          "logprob": -0.42895508,
           "text": "?"
         }
       ],
@@ -44318,61 +29524,61 @@
       "tokens": [
         {
           "id": 13,
-          "logprob": -0.0076828003,
+          "logprob": -0.007621765,
           "special": false,
           "text": "\n"
         },
         {
           "id": 13,
-          "logprob": -0.20092773,
+          "logprob": -0.20275879,
           "special": false,
           "text": "\n"
         },
         {
           "id": 16114,
-          "logprob": -1.2587891,
+          "logprob": -1.2578125,
           "special": false,
           "text": "Once"
         },
         {
           "id": 3714,
-          "logprob": -0.20861816,
+          "logprob": -0.2084961,
           "special": false,
           "text": " upon"
         },
         {
           "id": 264,
-          "logprob": -0.0017719269,
+          "logprob": -0.0017738342,
           "special": false,
           "text": " a"
         },
         {
           "id": 727,
-          "logprob": -0.011909485,
+          "logprob": -0.011932373,
           "special": false,
           "text": " time"
         },
         {
           "id": 28725,
-          "logprob": -0.17529297,
+          "logprob": -0.17297363,
           "special": false,
           "text": ","
         },
         {
           "id": 736,
-          "logprob": -0.9082031,
+          "logprob": -0.9057617,
           "special": false,
           "text": " there"
         },
         {
           "id": 403,
-          "logprob": -0.057525635,
+          "logprob": -0.05758667,
           "special": false,
           "text": " was"
         },
         {
           "id": 264,
-          "logprob": -0.009651184,
+          "logprob": -0.00970459,
           "special": false,
           "text": " a"
         }
@@ -44394,7 +29600,7 @@
         },
         {
           "id": 1247,
-          "logprob": -2.3886719,
+          "logprob": -2.390625,
           "text": "User"
         },
         {
@@ -44409,12 +29615,12 @@
         },
         {
           "id": 32000,
-          "logprob": -10.671875,
+          "logprob": -10.6953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.7109375,
+          "logprob": -15.828125,
           "text": "<image>"
         },
         {
@@ -44424,82 +29630,82 @@
         },
         {
           "id": 32000,
-          "logprob": -10.0234375,
+          "logprob": -10.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.1328125,
+          "logprob": -10.140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.421875,
+          "logprob": -10.3125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.90625,
+          "logprob": -12.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.59375,
+          "logprob": -15.4921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.828125,
+          "logprob": -13.8671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.390625,
+          "logprob": -11.3671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.1171875,
+          "logprob": -10.1015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.1640625,
+          "logprob": -10.1796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.234375,
+          "logprob": -10.2421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.3984375,
+          "logprob": -10.4609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.015625,
+          "logprob": -14.34375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0859375,
+          "logprob": -13.1796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2734375,
+          "logprob": -13.328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.359375,
+          "logprob": -14.3515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0390625,
+          "logprob": -11.0546875,
           "text": "<image>"
         },
         {
@@ -44509,12 +29715,12 @@
         },
         {
           "id": 32000,
-          "logprob": -10.5234375,
+          "logprob": -10.515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.4765625,
+          "logprob": -10.4453125,
           "text": "<image>"
         },
         {
@@ -44524,22 +29730,22 @@
         },
         {
           "id": 32000,
-          "logprob": -13.6171875,
+          "logprob": -13.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.359375,
+          "logprob": -11.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.8359375,
+          "logprob": -10.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -17.34375,
+          "logprob": -17.234375,
           "text": "<image>"
         },
         {
@@ -44554,12 +29760,12 @@
         },
         {
           "id": 32000,
-          "logprob": -10.640625,
+          "logprob": -10.6484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -18.390625,
+          "logprob": -17.984375,
           "text": "<image>"
         },
         {
@@ -44569,17 +29775,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.5625,
+          "logprob": -11.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -9.875,
+          "logprob": -9.8671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.7734375,
+          "logprob": -10.7578125,
           "text": "<image>"
         },
         {
@@ -44589,32 +29795,32 @@
         },
         {
           "id": 32000,
-          "logprob": -10.96875,
+          "logprob": -10.984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.609375,
+          "logprob": -10.625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.09375,
+          "logprob": -11.109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.5078125,
+          "logprob": -10.5,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4453125,
+          "logprob": -11.4609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.59375,
+          "logprob": -13.09375,
           "text": "<image>"
         },
         {
@@ -44624,132 +29830,87 @@
         },
         {
           "id": 32000,
-          "logprob": -10.5625,
+          "logprob": -10.5390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.640625,
+          "logprob": -10.625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9765625,
+          "logprob": -10.9296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.765625,
+          "logprob": -11.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3671875,
+          "logprob": -12.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0234375,
+          "logprob": -12.125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.59375,
+          "logprob": -10.578125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.7421875,
+          "logprob": -10.7265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0625,
+          "logprob": -11.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.3828125,
+          "logprob": -10.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.171875,
+          "logprob": -11.1796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0234375,
+          "logprob": -11.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -18.40625,
+          "logprob": -18.703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9921875,
+          "logprob": -11.015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.7109375,
+          "logprob": -15.6796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.15625,
+          "logprob": -12.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.140625,
+          "logprob": -10.4140625,
           "text": "<image>"
         },
         {
@@ -44759,12 +29920,42 @@
         },
         {
           "id": 32000,
-          "logprob": -10.7890625,
+          "logprob": -11.8046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4296875,
+          "logprob": -10.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.140625,
           "text": "<image>"
         },
         {
@@ -44774,12 +29965,27 @@
         },
         {
           "id": 32000,
-          "logprob": -10.796875,
+          "logprob": -10.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1640625,
+          "logprob": -11.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.15625,
           "text": "<image>"
         },
         {
@@ -44789,27 +29995,27 @@
         },
         {
           "id": 32000,
-          "logprob": -15.4453125,
+          "logprob": -15.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.2109375,
+          "logprob": -10.2265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.09375,
+          "logprob": -11.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.6796875,
+          "logprob": -14.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.3671875,
+          "logprob": -10.375,
           "text": "<image>"
         },
         {
@@ -44824,22 +30030,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.484375,
+          "logprob": -12.4921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.09375,
+          "logprob": -12.1015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.96875,
+          "logprob": -11.0859375,
           "text": "<image>"
         },
         {
@@ -44847,6 +30048,11 @@
           "logprob": -10.9765625,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -10.921875,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -11.0234375,
@@ -44862,11 +30068,6 @@
           "logprob": -11.1484375,
           "text": "<image>"
         },
-        {
-          "id": 32000,
-          "logprob": -10.703125,
-          "text": "<image>"
-        },
         {
           "id": 32000,
           "logprob": -10.71875,
@@ -44874,17 +30075,22 @@
         },
         {
           "id": 32000,
-          "logprob": -11.6875,
+          "logprob": -10.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.8984375,
+          "logprob": -11.6796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2890625,
+          "logprob": -10.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.375,
           "text": "<image>"
         },
         {
@@ -44894,12 +30100,12 @@
         },
         {
           "id": 32000,
-          "logprob": -10.640625,
+          "logprob": -10.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.7109375,
+          "logprob": -10.7265625,
           "text": "<image>"
         },
         {
@@ -44914,17 +30120,17 @@
         },
         {
           "id": 32000,
-          "logprob": -10.6875,
+          "logprob": -10.6796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.84375,
+          "logprob": -11.828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5078125,
+          "logprob": -11.5390625,
           "text": "<image>"
         },
         {
@@ -44934,22 +30140,22 @@
         },
         {
           "id": 32000,
-          "logprob": -10.9609375,
+          "logprob": -10.9765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4921875,
+          "logprob": -12.5,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5546875,
+          "logprob": -11.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.2265625,
+          "logprob": -11.2421875,
           "text": "<image>"
         },
         {
@@ -44959,7 +30165,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.21875,
+          "logprob": -11.2265625,
           "text": "<image>"
         },
         {
@@ -44969,52 +30175,52 @@
         },
         {
           "id": 32000,
-          "logprob": -11.0,
+          "logprob": -10.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0234375,
+          "logprob": -12.046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.09375,
+          "logprob": -11.109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.046875,
+          "logprob": -14.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.921875,
+          "logprob": -10.8984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9609375,
+          "logprob": -10.953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9140625,
+          "logprob": -10.9296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8125,
+          "logprob": -11.78125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.140625,
+          "logprob": -15.1484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9609375,
+          "logprob": -10.96875,
           "text": "<image>"
         },
         {
@@ -45024,72 +30230,72 @@
         },
         {
           "id": 32000,
-          "logprob": -10.8828125,
+          "logprob": -10.890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9609375,
+          "logprob": -10.9453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.46875,
+          "logprob": -11.453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5234375,
+          "logprob": -13.09375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.328125,
+          "logprob": -12.484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.375,
+          "logprob": -11.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3515625,
+          "logprob": -12.3359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.578125,
+          "logprob": -11.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5078125,
+          "logprob": -12.5,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.640625,
+          "logprob": -12.6328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1953125,
+          "logprob": -11.203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9921875,
+          "logprob": -11.9765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.921875,
+          "logprob": -10.9296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.7578125,
+          "logprob": -10.7421875,
           "text": "<image>"
         },
         {
@@ -45099,12 +30305,12 @@
         },
         {
           "id": 32000,
-          "logprob": -15.015625,
+          "logprob": -15.0234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7734375,
+          "logprob": -12.75,
           "text": "<image>"
         },
         {
@@ -45119,27 +30325,27 @@
         },
         {
           "id": 32000,
-          "logprob": -11.2890625,
+          "logprob": -11.265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.34375,
+          "logprob": -13.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.6953125,
+          "logprob": -10.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.59375,
+          "logprob": -10.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6015625,
+          "logprob": -11.7578125,
           "text": "<image>"
         },
         {
@@ -45154,7 +30360,7 @@
         },
         {
           "id": 32000,
-          "logprob": -10.5390625,
+          "logprob": -10.5859375,
           "text": "<image>"
         },
         {
@@ -45164,12 +30370,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.9765625,
+          "logprob": -17.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4609375,
+          "logprob": -11.4765625,
           "text": "<image>"
         },
         {
@@ -45179,12 +30385,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.015625,
+          "logprob": -12.0078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.84375,
+          "logprob": -12.8359375,
           "text": "<image>"
         },
         {
@@ -45194,12 +30400,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.4375,
+          "logprob": -12.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.671875,
+          "logprob": -12.6875,
           "text": "<image>"
         },
         {
@@ -45209,57 +30415,27 @@
         },
         {
           "id": 32000,
-          "logprob": -12.875,
+          "logprob": -12.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2578125,
+          "logprob": -12.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.359375,
+          "logprob": -10.3359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.375,
+          "logprob": -13.46875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.7421875,
+          "logprob": -11.8671875,
           "text": "<image>"
         },
         {
@@ -45269,7 +30445,32 @@
         },
         {
           "id": 32000,
-          "logprob": -11.0390625,
+          "logprob": -11.0078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8203125,
           "text": "<image>"
         },
         {
@@ -45279,17 +30480,22 @@
         },
         {
           "id": 32000,
-          "logprob": -10.6171875,
+          "logprob": -10.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2421875,
+          "logprob": -10.6328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.8359375,
+          "logprob": -12.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8203125,
           "text": "<image>"
         },
         {
@@ -45309,127 +30515,62 @@
         },
         {
           "id": 32000,
-          "logprob": -11.0,
+          "logprob": -10.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7734375,
+          "logprob": -12.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2109375,
+          "logprob": -12.21875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4296875,
+          "logprob": -11.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2421875,
+          "logprob": -12.828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3046875,
+          "logprob": -11.328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.2890625,
+          "logprob": -10.296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.8203125,
+          "logprob": -10.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9140625,
+          "logprob": -11.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.2421875,
+          "logprob": -11.21875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.234375,
+          "logprob": -11.2109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4375,
+          "logprob": -11.5078125,
           "text": "<image>"
         },
         {
@@ -45439,27 +30580,12 @@
         },
         {
           "id": 32000,
-          "logprob": -14.203125,
+          "logprob": -10.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.59375,
+          "logprob": -10.3515625,
           "text": "<image>"
         },
         {
@@ -45469,102 +30595,22 @@
         },
         {
           "id": 32000,
-          "logprob": -10.90625,
+          "logprob": -10.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.2578125,
+          "logprob": -10.828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.921875,
+          "logprob": -11.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.890625,
+          "logprob": -11.9375,
           "text": "<image>"
         },
         {
@@ -45574,112 +30620,32 @@
         },
         {
           "id": 32000,
-          "logprob": -10.9609375,
+          "logprob": -12.3046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.7890625,
+          "logprob": -11.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4453125,
+          "logprob": -12.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.390625,
+          "logprob": -13.3984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1640625,
+          "logprob": -11.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0703125,
+          "logprob": -14.1875,
           "text": "<image>"
         },
         {
@@ -45689,72 +30655,162 @@
         },
         {
           "id": 32000,
-          "logprob": -12.859375,
+          "logprob": -12.421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1875,
+          "logprob": -10.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.171875,
+          "logprob": -10.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6640625,
+          "logprob": -10.6015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1953125,
+          "logprob": -10.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1328125,
+          "logprob": -11.2265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1875,
+          "logprob": -10.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9453125,
+          "logprob": -10.9765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8515625,
+          "logprob": -12.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.203125,
+          "logprob": -10.8671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1796875,
+          "logprob": -10.8515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9609375,
+          "logprob": -11.8984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.703125,
+          "logprob": -10.7421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8515625,
+          "logprob": -13.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1796875,
           "text": "<image>"
         },
         {
@@ -45762,11 +30818,161 @@
           "logprob": -11.75,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -10.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.640625,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -11.8359375,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -11.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.828125,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -12.1953125,
@@ -45774,32 +30980,32 @@
         },
         {
           "id": 32000,
-          "logprob": -12.5078125,
+          "logprob": -12.4921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.546875,
+          "logprob": -11.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.078125,
+          "logprob": -12.0859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.2421875,
+          "logprob": -11.265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6640625,
+          "logprob": -11.6953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2109375,
+          "logprob": -12.21875,
           "text": "<image>"
         },
         {
@@ -45819,27 +31025,7 @@
         },
         {
           "id": 32000,
-          "logprob": -15.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3046875,
+          "logprob": -15.03125,
           "text": "<image>"
         },
         {
@@ -45849,32 +31035,52 @@
         },
         {
           "id": 32000,
-          "logprob": -12.2265625,
+          "logprob": -11.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9453125,
+          "logprob": -13.1171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4921875,
+          "logprob": -12.296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.8828125,
+          "logprob": -12.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3125,
+          "logprob": -12.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8984375,
+          "logprob": -12.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9140625,
           "text": "<image>"
         },
         {
@@ -45884,12 +31090,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8125,
+          "logprob": -11.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.90625,
+          "logprob": -11.9140625,
           "text": "<image>"
         },
         {
@@ -45909,12 +31115,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.140625,
+          "logprob": -13.1484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9765625,
+          "logprob": -11.9921875,
           "text": "<image>"
         },
         {
@@ -45929,7 +31135,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.84375,
+          "logprob": -11.8359375,
           "text": "<image>"
         },
         {
@@ -45944,17 +31150,22 @@
         },
         {
           "id": 32000,
-          "logprob": -13.1953125,
+          "logprob": -13.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.875,
+          "logprob": -11.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.6015625,
+          "logprob": -13.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6484375,
           "text": "<image>"
         },
         {
@@ -45964,27 +31175,22 @@
         },
         {
           "id": 32000,
-          "logprob": -13.671875,
+          "logprob": -11.4609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5390625,
+          "logprob": -13.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.59375,
+          "logprob": -12.4375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5703125,
+          "logprob": -14.6328125,
           "text": "<image>"
         },
         {
@@ -45994,57 +31200,57 @@
         },
         {
           "id": 32000,
-          "logprob": -13.4140625,
+          "logprob": -13.3984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7890625,
+          "logprob": -11.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6328125,
+          "logprob": -11.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.4296875,
+          "logprob": -14.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.53125,
+          "logprob": -13.5390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.515625,
+          "logprob": -14.5078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7265625,
+          "logprob": -12.7109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.609375,
+          "logprob": -11.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.171875,
+          "logprob": -13.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.109375,
+          "logprob": -13.09375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8828125,
+          "logprob": -12.875,
           "text": "<image>"
         },
         {
@@ -46054,37 +31260,37 @@
         },
         {
           "id": 32000,
-          "logprob": -12.671875,
+          "logprob": -12.6484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.7109375,
+          "logprob": -13.6953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4296875,
+          "logprob": -12.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.296875,
+          "logprob": -12.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1796875,
+          "logprob": -13.109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2421875,
+          "logprob": -12.1796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.828125,
+          "logprob": -14.875,
           "text": "<image>"
         },
         {
@@ -46094,7 +31300,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.3359375,
+          "logprob": -11.34375,
           "text": "<image>"
         },
         {
@@ -46114,67 +31320,7 @@
         },
         {
           "id": 32000,
-          "logprob": -15.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0,
+          "logprob": -15.0859375,
           "text": "<image>"
         },
         {
@@ -46184,57 +31330,117 @@
         },
         {
           "id": 32000,
-          "logprob": -16.21875,
+          "logprob": -11.875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6015625,
+          "logprob": -13.25,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.796875,
+          "logprob": -13.5078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7265625,
+          "logprob": -13.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9453125,
+          "logprob": -14.265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9765625,
+          "logprob": -14.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.71875,
+          "logprob": -13.2421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.6953125,
+          "logprob": -12.28125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3359375,
+          "logprob": -13.8046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3203125,
+          "logprob": -12.703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6328125,
+          "logprob": -12.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.609375,
           "text": "<image>"
         },
         {
@@ -46244,17 +31450,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1875,
+          "logprob": -14.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.046875,
+          "logprob": -12.0234375,
           "text": "<image>"
         },
         {
@@ -46269,17 +31475,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.8828125,
+          "logprob": -12.890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.296875,
+          "logprob": -12.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2109375,
+          "logprob": -12.21875,
           "text": "<image>"
         },
         {
@@ -46289,27 +31495,27 @@
         },
         {
           "id": 32000,
-          "logprob": -15.1875,
+          "logprob": -15.1953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5390625,
+          "logprob": -11.546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.421875,
+          "logprob": -14.4375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2890625,
+          "logprob": -12.265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2265625,
+          "logprob": -12.21875,
           "text": "<image>"
         },
         {
@@ -46324,12 +31530,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.5859375,
+          "logprob": -13.5390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.859375,
+          "logprob": -11.8515625,
           "text": "<image>"
         },
         {
@@ -46339,12 +31545,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.1015625,
+          "logprob": -13.046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.9453125,
+          "logprob": -13.953125,
           "text": "<image>"
         },
         {
@@ -46359,47 +31565,47 @@
         },
         {
           "id": 32000,
-          "logprob": -12.734375,
+          "logprob": -12.7109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.203125,
+          "logprob": -12.1796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.59375,
+          "logprob": -12.578125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3984375,
+          "logprob": -11.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5,
+          "logprob": -13.4609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.765625,
+          "logprob": -11.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2265625,
+          "logprob": -13.21875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7578125,
+          "logprob": -11.75,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.3515625,
+          "logprob": -15.34375,
           "text": "<image>"
         },
         {
@@ -46414,22 +31620,42 @@
         },
         {
           "id": 32000,
-          "logprob": -13.3671875,
+          "logprob": -13.3515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.90625,
+          "logprob": -11.8828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5625,
+          "logprob": -12.578125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3203125,
+          "logprob": -13.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.125,
           "text": "<image>"
         },
         {
@@ -46439,27 +31665,7 @@
         },
         {
           "id": 32000,
-          "logprob": -10.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9921875,
+          "logprob": -11.984375,
           "text": "<image>"
         },
         {
@@ -46474,27 +31680,27 @@
         },
         {
           "id": 32000,
-          "logprob": -12.9765625,
+          "logprob": -12.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4921875,
+          "logprob": -12.484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0625,
+          "logprob": -12.046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1796875,
+          "logprob": -12.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3359375,
+          "logprob": -13.296875,
           "text": "<image>"
         },
         {
@@ -46504,32 +31710,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.375,
+          "logprob": -12.3671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.984375,
+          "logprob": -13.1328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9453125,
+          "logprob": -13.625,
           "text": "<image>"
         },
         {
@@ -46539,87 +31730,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.3203125,
+          "logprob": -11.75,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3125,
+          "logprob": -11.9140625,
           "text": "<image>"
         },
         {
@@ -46629,12 +31745,102 @@
         },
         {
           "id": 32000,
-          "logprob": -11.3984375,
+          "logprob": -12.3359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.84375,
+          "logprob": -12.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8359375,
           "text": "<image>"
         },
         {
@@ -46644,52 +31850,52 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6328125,
+          "logprob": -12.6171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.875,
+          "logprob": -13.84375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.109375,
+          "logprob": -12.09375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1171875,
+          "logprob": -12.140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4921875,
+          "logprob": -11.5078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.2890625,
+          "logprob": -11.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.15625,
+          "logprob": -12.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.59375,
+          "logprob": -13.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8046875,
+          "logprob": -11.7890625,
           "text": "<image>"
         },
         {
@@ -46699,22 +31905,22 @@
         },
         {
           "id": 32000,
-          "logprob": -13.1015625,
+          "logprob": -13.0625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2265625,
+          "logprob": -13.1875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2109375,
+          "logprob": -13.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4609375,
+          "logprob": -12.46875,
           "text": "<image>"
         },
         {
@@ -46724,12 +31930,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
+          "logprob": -12.6953125,
           "text": "<image>"
         },
         {
@@ -46737,6 +31938,11 @@
           "logprob": -11.6875,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -11.6953125,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -11.7109375,
@@ -46744,27 +31950,27 @@
         },
         {
           "id": 32000,
-          "logprob": -15.390625,
+          "logprob": -15.3359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.953125,
+          "logprob": -11.9453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.140625,
+          "logprob": -16.125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.4921875,
+          "logprob": -15.4609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.9296875,
+          "logprob": -13.921875,
           "text": "<image>"
         },
         {
@@ -46779,7 +31985,7 @@
         },
         {
           "id": 32000,
-          "logprob": -15.984375,
+          "logprob": -16.0,
           "text": "<image>"
         },
         {
@@ -46789,37 +31995,37 @@
         },
         {
           "id": 32000,
-          "logprob": -12.8671875,
+          "logprob": -12.828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4921875,
+          "logprob": -12.5078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7421875,
+          "logprob": -11.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1875,
+          "logprob": -14.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3515625,
+          "logprob": -11.34375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.71875,
+          "logprob": -11.7265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0625,
+          "logprob": -13.046875,
           "text": "<image>"
         },
         {
@@ -46829,12 +32035,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.421875,
+          "logprob": -12.4375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.2734375,
+          "logprob": -14.265625,
           "text": "<image>"
         },
         {
@@ -46844,77 +32050,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.390625,
+          "logprob": -12.6953125,
           "text": "<image>"
         },
         {
@@ -46924,27 +32060,97 @@
         },
         {
           "id": 32000,
-          "logprob": -13.078125,
+          "logprob": -12.984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4296875,
+          "logprob": -13.2890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.828125,
+          "logprob": -13.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8046875,
+          "logprob": -11.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6484375,
+          "logprob": -10.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.625,
           "text": "<image>"
         },
         {
@@ -46959,27 +32165,27 @@
         },
         {
           "id": 32000,
-          "logprob": -14.921875,
+          "logprob": -14.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.578125,
+          "logprob": -12.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.78125,
+          "logprob": -12.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3984375,
+          "logprob": -11.375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.0546875,
+          "logprob": -14.03125,
           "text": "<image>"
         },
         {
@@ -46999,12 +32205,12 @@
         },
         {
           "id": 32000,
-          "logprob": -14.5234375,
+          "logprob": -14.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.625,
+          "logprob": -12.609375,
           "text": "<image>"
         },
         {
@@ -47014,12 +32220,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.6015625,
+          "logprob": -13.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.28125,
+          "logprob": -13.2109375,
           "text": "<image>"
         },
         {
@@ -47039,17 +32245,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.0,
+          "logprob": -12.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6640625,
+          "logprob": -12.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.46875,
+          "logprob": -11.484375,
           "text": "<image>"
         },
         {
@@ -47059,12 +32265,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8828125,
+          "logprob": -11.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1015625,
+          "logprob": -13.078125,
           "text": "<image>"
         },
         {
@@ -47079,7 +32285,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.5234375,
+          "logprob": -11.546875,
           "text": "<image>"
         },
         {
@@ -47094,7 +32300,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.4921875,
+          "logprob": -12.5,
           "text": "<image>"
         },
         {
@@ -47109,122 +32315,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.7890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1484375,
+          "logprob": -12.078125,
           "text": "<image>"
         },
         {
@@ -47234,132 +32325,102 @@
         },
         {
           "id": 32000,
-          "logprob": -13.3828125,
+          "logprob": -13.2265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3203125,
+          "logprob": -14.8046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.34375,
+          "logprob": -12.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -11.8671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.140625,
+          "logprob": -13.1484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9921875,
+          "logprob": -12.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.15625,
+          "logprob": -10.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.84375,
+          "logprob": -14.6875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.21875,
+          "logprob": -15.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.46875,
+          "logprob": -15.453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1484375,
+          "logprob": -11.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.515625,
+          "logprob": -14.484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.84375,
+          "logprob": -12.4375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.015625,
+          "logprob": -16.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.28125,
+          "logprob": -14.875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6015625,
+          "logprob": -16.109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3984375,
+          "logprob": -13.2890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.375,
+          "logprob": -12.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5625,
+          "logprob": -11.8359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -9.9375,
+          "logprob": -12.9921875,
           "text": "<image>"
         },
         {
@@ -47369,17 +32430,162 @@
         },
         {
           "id": 32000,
-          "logprob": -12.203125,
+          "logprob": -12.1484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.2890625,
+          "logprob": -13.84375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1953125,
+          "logprob": -13.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -9.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.15625,
           "text": "<image>"
         },
         {
@@ -47392,21 +32598,6 @@
           "logprob": -12.28125,
           "text": "<image>"
         },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
         {
           "id": 32000,
           "logprob": -11.8984375,
@@ -47414,52 +32605,67 @@
         },
         {
           "id": 32000,
-          "logprob": -10.5859375,
+          "logprob": -14.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9921875,
+          "logprob": -14.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9921875,
+          "logprob": -11.8671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1015625,
+          "logprob": -10.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.5390625,
+          "logprob": -13.0,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.2578125,
+          "logprob": -12.8984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1171875,
+          "logprob": -12.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2421875,
+          "logprob": -15.765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5,
+          "logprob": -15.203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7265625,
+          "logprob": -14.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8359375,
           "text": "<image>"
         },
         {
@@ -47469,7 +32675,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.0,
+          "logprob": -11.0078125,
           "text": "<image>"
         },
         {
@@ -47479,22 +32685,22 @@
         },
         {
           "id": 32000,
-          "logprob": -13.28125,
+          "logprob": -13.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7734375,
+          "logprob": -11.7578125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0625,
+          "logprob": -13.09375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.671875,
+          "logprob": -13.6796875,
           "text": "<image>"
         },
         {
@@ -47504,7 +32710,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.3828125,
+          "logprob": -12.421875,
           "text": "<image>"
         },
         {
@@ -47514,22 +32720,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.25,
+          "logprob": -12.2890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.9453125,
+          "logprob": -11.921875,
           "text": "<image>"
         },
         {
@@ -47539,7 +32735,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.9375,
+          "logprob": -15.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
           "text": "<image>"
         },
         {
@@ -47549,37 +32755,37 @@
         },
         {
           "id": 32000,
-          "logprob": -11.3203125,
+          "logprob": -11.328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4921875,
+          "logprob": -11.5,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.359375,
+          "logprob": -11.3671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3359375,
+          "logprob": -11.3515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0546875,
+          "logprob": -12.0625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.359375,
+          "logprob": -12.3359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -9.6953125,
+          "logprob": -9.7109375,
           "text": "<image>"
         },
         {
@@ -47589,27 +32795,27 @@
         },
         {
           "id": 32000,
-          "logprob": -14.3203125,
+          "logprob": -14.2421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9609375,
+          "logprob": -11.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0859375,
+          "logprob": -12.09375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2109375,
+          "logprob": -12.1796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4921875,
+          "logprob": -11.5234375,
           "text": "<image>"
         },
         {
@@ -47624,7 +32830,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.0546875,
           "text": "<image>"
         },
         {
@@ -47639,22 +32845,22 @@
         },
         {
           "id": 32000,
-          "logprob": -11.9921875,
+          "logprob": -12.0,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2265625,
+          "logprob": -12.1953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9921875,
+          "logprob": -12.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6796875,
+          "logprob": -12.7109375,
           "text": "<image>"
         },
         {
@@ -47664,7 +32870,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.5703125,
+          "logprob": -13.5546875,
           "text": "<image>"
         },
         {
@@ -47679,17 +32885,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.9375,
+          "logprob": -11.953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.9453125,
+          "logprob": -13.8046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.984375,
+          "logprob": -11.9921875,
           "text": "<image>"
         },
         {
@@ -47699,17 +32905,17 @@
         },
         {
           "id": 32000,
-          "logprob": -10.03125,
+          "logprob": -10.015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7265625,
+          "logprob": -11.71875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.7890625,
+          "logprob": -13.8125,
           "text": "<image>"
         },
         {
@@ -47719,42 +32925,42 @@
         },
         {
           "id": 32000,
-          "logprob": -12.671875,
+          "logprob": -12.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.59375,
+          "logprob": -12.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1171875,
+          "logprob": -12.1015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4609375,
+          "logprob": -12.453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3046875,
+          "logprob": -12.2890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4765625,
+          "logprob": -12.46875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6328125,
+          "logprob": -12.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.03125,
+          "logprob": -12.015625,
           "text": "<image>"
         },
         {
@@ -47764,132 +32970,32 @@
         },
         {
           "id": 32000,
-          "logprob": -14.5078125,
+          "logprob": -14.421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2265625,
+          "logprob": -13.25,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.546875,
+          "logprob": -16.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.4765625,
+          "logprob": -14.484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.71875,
+          "logprob": -13.7421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.7890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.671875,
+          "logprob": -13.46875,
           "text": "<image>"
         },
         {
@@ -47899,57 +33005,97 @@
         },
         {
           "id": 32000,
-          "logprob": -13.40625,
+          "logprob": -12.8671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.265625,
+          "logprob": -12.4921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.015625,
+          "logprob": -13.2265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5390625,
+          "logprob": -13.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.0234375,
+          "logprob": -12.4375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1796875,
+          "logprob": -10.765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6875,
+          "logprob": -13.109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.9453125,
+          "logprob": -12.8515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6875,
+          "logprob": -14.7421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.78125,
+          "logprob": -12.84375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.9609375,
+          "logprob": -12.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6484375,
           "text": "<image>"
         },
         {
@@ -47959,27 +33105,87 @@
         },
         {
           "id": 32000,
-          "logprob": -16.140625,
+          "logprob": -13.4296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4609375,
+          "logprob": -13.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.7265625,
+          "logprob": -12.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.609375,
+          "logprob": -11.5234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.234375,
+          "logprob": -15.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.25,
           "text": "<image>"
         },
         {
@@ -47989,7 +33195,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.4921875,
+          "logprob": -13.5,
           "text": "<image>"
         },
         {
@@ -48014,12 +33220,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.15625,
+          "logprob": -13.1484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -17.046875,
+          "logprob": -17.0625,
           "text": "<image>"
         },
         {
@@ -48029,12 +33235,12 @@
         },
         {
           "id": 32000,
-          "logprob": -14.265625,
+          "logprob": -14.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6328125,
+          "logprob": -12.625,
           "text": "<image>"
         },
         {
@@ -48044,92 +33250,37 @@
         },
         {
           "id": 32000,
-          "logprob": -13.9375,
+          "logprob": -13.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0390625,
+          "logprob": -13.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.46875,
+          "logprob": -10.5,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1640625,
+          "logprob": -13.1953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.59375,
+          "logprob": -13.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.390625,
+          "logprob": -13.3828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8125,
+          "logprob": -14.0625,
           "text": "<image>"
         },
         {
@@ -48137,6 +33288,61 @@
           "logprob": -13.5234375,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -14.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.53125,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -11.4140625,
@@ -48144,17 +33350,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.46875,
+          "logprob": -11.4609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.09375,
+          "logprob": -13.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.34375,
+          "logprob": -12.3359375,
           "text": "<image>"
         },
         {
@@ -48164,32 +33370,32 @@
         },
         {
           "id": 32000,
-          "logprob": -12.7578125,
+          "logprob": -12.75,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.8359375,
+          "logprob": -14.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0859375,
+          "logprob": -13.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1640625,
+          "logprob": -14.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0625,
+          "logprob": -13.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.5078125,
+          "logprob": -14.4921875,
           "text": "<image>"
         },
         {
@@ -48199,7 +33405,7 @@
         },
         {
           "id": 32000,
-          "logprob": -14.6953125,
+          "logprob": -14.703125,
           "text": "<image>"
         },
         {
@@ -48209,32 +33415,32 @@
         },
         {
           "id": 32000,
-          "logprob": -16.328125,
+          "logprob": -16.3125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.9921875,
+          "logprob": -14.9765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.8515625,
+          "logprob": -13.8046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.40625,
+          "logprob": -15.375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4296875,
+          "logprob": -13.3984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.5859375,
+          "logprob": -14.578125,
           "text": "<image>"
         },
         {
@@ -48249,7 +33455,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.3359375,
+          "logprob": -13.328125,
           "text": "<image>"
         },
         {
@@ -48264,37 +33470,52 @@
         },
         {
           "id": 32000,
-          "logprob": -16.890625,
+          "logprob": -16.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.328125,
+          "logprob": -13.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.78125,
+          "logprob": -13.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.34375,
+          "logprob": -13.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4921875,
+          "logprob": -13.4765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4296875,
+          "logprob": -13.390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.28125,
+          "logprob": -13.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5,
           "text": "<image>"
         },
         {
@@ -48304,7 +33525,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.75,
+          "logprob": -13.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6796875,
           "text": "<image>"
         },
         {
@@ -48314,32 +33545,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.328125,
+          "logprob": -13.3203125,
           "text": "<image>"
         },
         {
@@ -48349,47 +33555,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.640625,
+          "logprob": -13.6171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.15625,
+          "logprob": -15.140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0078125,
+          "logprob": -13.21875,
           "text": "<image>"
         },
         {
@@ -48399,27 +33575,57 @@
         },
         {
           "id": 32000,
-          "logprob": -11.625,
+          "logprob": -12.0234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8671875,
+          "logprob": -11.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.578125,
+          "logprob": -11.3125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8125,
+          "logprob": -10.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.9453125,
+          "logprob": -13.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.9765625,
           "text": "<image>"
         },
         {
@@ -48429,17 +33635,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.9765625,
+          "logprob": -12.984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.828125,
+          "logprob": -13.8359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7265625,
+          "logprob": -12.75,
           "text": "<image>"
         },
         {
@@ -48454,12 +33660,12 @@
         },
         {
           "id": 32000,
-          "logprob": -14.7578125,
+          "logprob": -14.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.5234375,
+          "logprob": -14.5078125,
           "text": "<image>"
         },
         {
@@ -48469,12 +33675,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.0390625,
+          "logprob": -13.046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4453125,
+          "logprob": -13.453125,
           "text": "<image>"
         },
         {
@@ -48489,17 +33695,17 @@
         },
         {
           "id": 32000,
-          "logprob": -14.0,
+          "logprob": -13.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5390625,
+          "logprob": -13.4921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0859375,
+          "logprob": -11.9765625,
           "text": "<image>"
         },
         {
@@ -48509,17 +33715,37 @@
         },
         {
           "id": 32000,
-          "logprob": -15.953125,
+          "logprob": -15.9609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.2265625,
+          "logprob": -14.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1484375,
+          "logprob": -13.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.640625,
           "text": "<image>"
         },
         {
@@ -48529,32 +33755,47 @@
         },
         {
           "id": 32000,
-          "logprob": -12.6875,
+          "logprob": -13.625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.796875,
+          "logprob": -12.890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.65625,
+          "logprob": -12.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6796875,
+          "logprob": -12.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.65625,
+          "logprob": -12.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.90625,
+          "logprob": -14.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0078125,
           "text": "<image>"
         },
         {
@@ -48564,52 +33805,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.6171875,
+          "logprob": -15.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7890625,
+          "logprob": -12.921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.375,
+          "logprob": -12.40625,
           "text": "<image>"
         },
         {
@@ -48624,27 +33830,27 @@
         },
         {
           "id": 32000,
-          "logprob": -11.375,
+          "logprob": -11.3671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0625,
+          "logprob": -11.0703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3046875,
+          "logprob": -11.28125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0390625,
+          "logprob": -11.046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3515625,
+          "logprob": -13.3203125,
           "text": "<image>"
         },
         {
@@ -48669,12 +33875,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.8359375,
+          "logprob": -13.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0234375,
+          "logprob": -12.0859375,
           "text": "<image>"
         },
         {
@@ -48684,12 +33890,7 @@
         },
         {
           "id": 32000,
-          "logprob": -14.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0546875,
+          "logprob": -14.0703125,
           "text": "<image>"
         },
         {
@@ -48699,7 +33900,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.0625,
+          "logprob": -14.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0390625,
           "text": "<image>"
         },
         {
@@ -48709,42 +33915,42 @@
         },
         {
           "id": 32000,
-          "logprob": -14.46875,
+          "logprob": -14.453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.703125,
+          "logprob": -14.6875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.296875,
+          "logprob": -14.3046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.8828125,
+          "logprob": -14.8671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1796875,
+          "logprob": -14.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2265625,
+          "logprob": -13.203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1171875,
+          "logprob": -13.109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9375,
+          "logprob": -11.8515625,
           "text": "<image>"
         },
         {
@@ -48759,12 +33965,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.71875,
+          "logprob": -12.7109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.734375,
+          "logprob": -13.7109375,
           "text": "<image>"
         },
         {
@@ -48774,32 +33980,32 @@
         },
         {
           "id": 32000,
-          "logprob": -12.46875,
+          "logprob": -12.4765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.15625,
+          "logprob": -12.140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.375,
+          "logprob": -12.3828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8984375,
+          "logprob": -12.875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.6953125,
+          "logprob": -13.6484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0625,
+          "logprob": -13.0390625,
           "text": "<image>"
         },
         {
@@ -48809,12 +34015,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.6796875,
+          "logprob": -12.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3515625,
+          "logprob": -12.34375,
           "text": "<image>"
         },
         {
@@ -48824,37 +34030,37 @@
         },
         {
           "id": 32000,
-          "logprob": -12.46875,
+          "logprob": -12.453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.03125,
+          "logprob": -12.015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.96875,
+          "logprob": -11.9765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.4609375,
+          "logprob": -14.46875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.328125,
+          "logprob": -13.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6484375,
+          "logprob": -12.71875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.65625,
+          "logprob": -10.6484375,
           "text": "<image>"
         },
         {
@@ -48874,22 +34080,22 @@
         },
         {
           "id": 32000,
-          "logprob": -12.6328125,
+          "logprob": -12.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.265625,
+          "logprob": -11.2578125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.734375,
+          "logprob": -11.7265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.84375,
+          "logprob": -10.8515625,
           "text": "<image>"
         },
         {
@@ -48899,32 +34105,32 @@
         },
         {
           "id": 32000,
-          "logprob": -13.109375,
+          "logprob": -13.140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7421875,
+          "logprob": -12.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.09375,
+          "logprob": -12.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.109375,
+          "logprob": -13.1328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.671875,
+          "logprob": -12.65625,
           "text": "<image>"
         },
         {
@@ -48934,27 +34140,27 @@
         },
         {
           "id": 32000,
-          "logprob": -12.6796875,
+          "logprob": -12.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.109375,
+          "logprob": -12.0859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.625,
+          "logprob": -13.609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8515625,
+          "logprob": -12.8359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3046875,
+          "logprob": -12.3125,
           "text": "<image>"
         },
         {
@@ -48964,12 +34170,12 @@
         },
         {
           "id": 32000,
-          "logprob": -14.0703125,
+          "logprob": -14.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.0,
+          "logprob": -13.984375,
           "text": "<image>"
         },
         {
@@ -48979,7 +34185,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.2421875,
+          "logprob": -13.2109375,
           "text": "<image>"
         },
         {
@@ -48989,42 +34195,42 @@
         },
         {
           "id": 32000,
-          "logprob": -13.7109375,
+          "logprob": -13.703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6015625,
+          "logprob": -12.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.796875,
+          "logprob": -15.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.90625,
+          "logprob": -11.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.375,
+          "logprob": -12.3671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4765625,
+          "logprob": -11.46875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.015625,
+          "logprob": -12.0,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.328125,
+          "logprob": -11.3359375,
           "text": "<image>"
         },
         {
@@ -49039,17 +34245,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.8203125,
+          "logprob": -13.828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7578125,
+          "logprob": -11.75,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.484375,
+          "logprob": -11.46875,
           "text": "<image>"
         },
         {
@@ -49064,17 +34270,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.640625,
+          "logprob": -11.625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5859375,
+          "logprob": -11.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.2578125,
+          "logprob": -15.2265625,
           "text": "<image>"
         },
         {
@@ -49084,17 +34290,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.0078125,
+          "logprob": -13.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.3828125,
+          "logprob": -15.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.203125,
+          "logprob": -11.1875,
           "text": "<image>"
         },
         {
@@ -49104,7 +34310,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.0703125,
+          "logprob": -11.0625,
           "text": "<image>"
         },
         {
@@ -49114,7 +34320,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.4453125,
+          "logprob": -13.453125,
           "text": "<image>"
         },
         {
@@ -49124,27 +34330,27 @@
         },
         {
           "id": 32000,
-          "logprob": -11.5078125,
+          "logprob": -11.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.015625,
+          "logprob": -12.0078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.15625,
+          "logprob": -11.1875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8359375,
+          "logprob": -11.890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.359375,
+          "logprob": -11.3125,
           "text": "<image>"
         },
         {
@@ -49159,22 +34365,22 @@
         },
         {
           "id": 32000,
-          "logprob": -12.4453125,
+          "logprob": -12.4921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.015625,
+          "logprob": -12.0703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.375,
+          "logprob": -13.3671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2734375,
+          "logprob": -13.28125,
           "text": "<image>"
         },
         {
@@ -49184,22 +34390,22 @@
         },
         {
           "id": 32000,
-          "logprob": -13.046875,
+          "logprob": -13.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0859375,
+          "logprob": -13.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1640625,
+          "logprob": -12.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4765625,
+          "logprob": -13.484375,
           "text": "<image>"
         },
         {
@@ -49209,7 +34415,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.7265625,
+          "logprob": -13.703125,
           "text": "<image>"
         },
         {
@@ -49219,262 +34425,17 @@
         },
         {
           "id": 32000,
-          "logprob": -14.3359375,
+          "logprob": -14.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.71875,
+          "logprob": -12.7109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.640625,
+          "logprob": -14.28125,
           "text": "<image>"
         },
         {
@@ -49484,52 +34445,57 @@
         },
         {
           "id": 32000,
-          "logprob": -13.453125,
+          "logprob": -13.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.4375,
+          "logprob": -15.203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1171875,
+          "logprob": -13.28125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4296875,
+          "logprob": -12.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.515625,
+          "logprob": -14.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.21875,
+          "logprob": -13.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8671875,
+          "logprob": -12.1171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8515625,
+          "logprob": -14.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.8671875,
+          "logprob": -14.0625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.59375,
+          "logprob": -12.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.953125,
           "text": "<image>"
         },
         {
@@ -49539,12 +34505,57 @@
         },
         {
           "id": 32000,
-          "logprob": -12.1640625,
+          "logprob": -11.6953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2265625,
+          "logprob": -12.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5234375,
           "text": "<image>"
         },
         {
@@ -49554,32 +34565,27 @@
         },
         {
           "id": 32000,
-          "logprob": -12.1015625,
+          "logprob": -11.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0234375,
+          "logprob": -11.2578125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4140625,
+          "logprob": -13.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.828125,
+          "logprob": -11.921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1953125,
+          "logprob": -13.03125,
           "text": "<image>"
         },
         {
@@ -49589,7 +34595,207 @@
         },
         {
           "id": 32000,
-          "logprob": -11.1328125,
+          "logprob": -14.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.4296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.140625,
           "text": "<image>"
         },
         {
@@ -49614,7 +34820,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.1015625,
+          "logprob": -11.1171875,
           "text": "<image>"
         },
         {
@@ -49624,7 +34830,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.15625,
+          "logprob": -11.1484375,
           "text": "<image>"
         },
         {
@@ -49634,17 +34840,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.4296875,
+          "logprob": -11.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.765625,
+          "logprob": -12.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.09375,
+          "logprob": -13.0859375,
           "text": "<image>"
         },
         {
@@ -49654,22 +34860,22 @@
         },
         {
           "id": 32000,
-          "logprob": -12.96875,
+          "logprob": -12.9609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.328125,
+          "logprob": -12.34375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8359375,
+          "logprob": -12.84375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.9609375,
+          "logprob": -13.9765625,
           "text": "<image>"
         },
         {
@@ -49679,7 +34885,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.46875,
+          "logprob": -13.484375,
           "text": "<image>"
         },
         {
@@ -49689,22 +34895,22 @@
         },
         {
           "id": 32000,
-          "logprob": -13.59375,
+          "logprob": -13.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.40625,
+          "logprob": -13.3828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.234375,
+          "logprob": -14.25,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.265625,
+          "logprob": -14.2578125,
           "text": "<image>"
         },
         {
@@ -49714,127 +34920,127 @@
         },
         {
           "id": 32000,
-          "logprob": -13.65625,
+          "logprob": -13.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.2734375,
+          "logprob": -14.2578125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.09375,
+          "logprob": -13.8671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.890625,
+          "logprob": -12.921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.640625,
+          "logprob": -15.6484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0234375,
+          "logprob": -13.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3828125,
+          "logprob": -12.34375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.71875,
+          "logprob": -12.7265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.5234375,
+          "logprob": -14.515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7734375,
+          "logprob": -12.75,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3203125,
+          "logprob": -12.265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.609375,
+          "logprob": -13.546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1640625,
+          "logprob": -13.1328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.203125,
+          "logprob": -12.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1875,
+          "logprob": -12.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.59375,
+          "logprob": -11.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6875,
+          "logprob": -11.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1796875,
+          "logprob": -12.15625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6484375,
+          "logprob": -11.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.421875,
+          "logprob": -11.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.34375,
+          "logprob": -11.390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1015625,
+          "logprob": -11.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9765625,
+          "logprob": -11.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.015625,
+          "logprob": -12.0078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0,
+          "logprob": -12.984375,
           "text": "<image>"
         },
         {
@@ -49844,7 +35050,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.09375,
           "text": "<image>"
         },
         {
@@ -49864,12 +35070,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.65625,
+          "logprob": -11.6484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.84375,
+          "logprob": -12.8125,
           "text": "<image>"
         },
         {
@@ -49879,17 +35085,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.6953125,
+          "logprob": -11.6796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.578125,
+          "logprob": -11.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.6796875,
+          "logprob": -13.6953125,
           "text": "<image>"
         },
         {
@@ -49904,7 +35110,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0078125,
+          "logprob": -12.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.296875,
           "text": "<image>"
         },
         {
@@ -49914,17 +35125,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.234375,
+          "logprob": -13.84375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8671875,
+          "logprob": -12.875,
           "text": "<image>"
         },
         {
@@ -49934,17 +35140,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.5078125,
+          "logprob": -13.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.953125,
+          "logprob": -13.9609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5078125,
+          "logprob": -12.46875,
           "text": "<image>"
         },
         {
@@ -49954,232 +35160,17 @@
         },
         {
           "id": 32000,
-          "logprob": -14.25,
+          "logprob": -14.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9140625,
+          "logprob": -12.921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4375,
+          "logprob": -14.2578125,
           "text": "<image>"
         },
         {
@@ -50189,62 +35180,62 @@
         },
         {
           "id": 32000,
-          "logprob": -13.78125,
+          "logprob": -14.3984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5703125,
+          "logprob": -15.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9765625,
+          "logprob": -13.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4765625,
+          "logprob": -12.3515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.125,
+          "logprob": -13.46875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0859375,
+          "logprob": -11.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9296875,
+          "logprob": -12.2890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.90625,
+          "logprob": -14.3515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0625,
+          "logprob": -13.140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0234375,
+          "logprob": -13.0234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.6953125,
+          "logprob": -12.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.890625,
+          "logprob": -11.8125,
           "text": "<image>"
         },
         {
@@ -50252,6 +35243,31 @@
           "logprob": -12.125,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -11.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.640625,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -12.4375,
@@ -50259,67 +35275,257 @@
         },
         {
           "id": 32000,
-          "logprob": -12.375,
+          "logprob": -11.46875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.28125,
+          "logprob": -11.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4921875,
+          "logprob": -12.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1640625,
+          "logprob": -11.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1953125,
+          "logprob": -11.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.4140625,
+          "logprob": -12.4453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6171875,
+          "logprob": -11.34375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.921875,
+          "logprob": -12.1015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6328125,
+          "logprob": -11.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2421875,
+          "logprob": -14.6015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0234375,
+          "logprob": -12.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6953125,
+          "logprob": -11.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1484375,
+          "logprob": -11.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1328125,
           "text": "<image>"
         },
         {
@@ -50329,27 +35535,27 @@
         },
         {
           "id": 32000,
-          "logprob": -11.7265625,
+          "logprob": -11.71875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6796875,
+          "logprob": -11.6875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4921875,
+          "logprob": -12.484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.875,
+          "logprob": -11.8828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5390625,
+          "logprob": -12.53125,
           "text": "<image>"
         },
         {
@@ -50364,27 +35570,27 @@
         },
         {
           "id": 32000,
-          "logprob": -13.71875,
+          "logprob": -13.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.78125,
+          "logprob": -14.8046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.578125,
+          "logprob": -11.609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5859375,
+          "logprob": -13.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.390625,
+          "logprob": -12.421875,
           "text": "<image>"
         },
         {
@@ -50394,17 +35600,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.65625,
+          "logprob": -13.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5,
+          "logprob": -12.5078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.109375,
+          "logprob": -13.125,
           "text": "<image>"
         },
         {
@@ -50414,37 +35620,37 @@
         },
         {
           "id": 32000,
-          "logprob": -12.2578125,
+          "logprob": -12.2890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7421875,
+          "logprob": -12.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1640625,
+          "logprob": -13.15625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9375,
+          "logprob": -11.9453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.265625,
+          "logprob": -12.28125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1640625,
+          "logprob": -12.1328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.8671875,
+          "logprob": -13.8203125,
           "text": "<image>"
         },
         {
@@ -50454,57 +35660,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.046875,
+          "logprob": -13.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.984375,
+          "logprob": -15.6953125,
           "text": "<image>"
         },
         {
@@ -50514,12 +35675,47 @@
         },
         {
           "id": 32000,
-          "logprob": -11.3984375,
+          "logprob": -14.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.796875,
+          "logprob": -13.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.234375,
           "text": "<image>"
         },
         {
@@ -50529,37 +35725,47 @@
         },
         {
           "id": 32000,
-          "logprob": -11.4296875,
+          "logprob": -11.828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9921875,
+          "logprob": -11.453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.59375,
+          "logprob": -11.453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8359375,
+          "logprob": -11.984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7421875,
+          "logprob": -11.609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.125,
+          "logprob": -11.8046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.875,
+          "logprob": -11.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8515625,
           "text": "<image>"
         },
         {
@@ -50569,27 +35775,27 @@
         },
         {
           "id": 32000,
-          "logprob": -10.796875,
+          "logprob": -10.8046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4765625,
+          "logprob": -11.578125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1484375,
+          "logprob": -11.15625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8046875,
+          "logprob": -12.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3828125,
+          "logprob": -11.3671875,
           "text": "<image>"
         },
         {
@@ -50609,7 +35815,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.5546875,
+          "logprob": -12.5625,
           "text": "<image>"
         },
         {
@@ -50619,367 +35825,57 @@
         },
         {
           "id": 32000,
-          "logprob": -10.40625,
+          "logprob": -10.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.59375,
+          "logprob": -12.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.03125,
+          "logprob": -12.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.03125,
+          "logprob": -12.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.65625,
+          "logprob": -12.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6328125,
+          "logprob": -11.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5,
+          "logprob": -13.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.078125,
+          "logprob": -12.0703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0234375,
+          "logprob": -13.015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.171875,
+          "logprob": -14.1484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.4609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.015625,
+          "logprob": -12.3671875,
           "text": "<image>"
         },
         {
@@ -50989,42 +35885,112 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0859375,
+          "logprob": -12.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.140625,
+          "logprob": -13.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.65625,
+          "logprob": -13.953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.890625,
+          "logprob": -12.4765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.59375,
+          "logprob": -14.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4375,
+          "logprob": -12.6015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4375,
+          "logprob": -13.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.109375,
+          "logprob": -15.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0546875,
           "text": "<image>"
         },
         {
@@ -51034,17 +36000,257 @@
         },
         {
           "id": 32000,
-          "logprob": -11.1015625,
+          "logprob": -11.9609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.203125,
+          "logprob": -14.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9921875,
+          "logprob": -12.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.1484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.96875,
           "text": "<image>"
         },
         {
@@ -51054,32 +36260,32 @@
         },
         {
           "id": 32000,
-          "logprob": -11.40625,
+          "logprob": -11.4296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.53125,
+          "logprob": -12.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.109375,
+          "logprob": -12.1015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3203125,
+          "logprob": -11.3125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.125,
+          "logprob": -11.1171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7578125,
+          "logprob": -11.75,
           "text": "<image>"
         },
         {
@@ -51094,7 +36300,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.109375,
+          "logprob": -11.1015625,
           "text": "<image>"
         },
         {
@@ -51114,67 +36320,67 @@
         },
         {
           "id": 32000,
-          "logprob": -12.609375,
+          "logprob": -12.6015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.65625,
+          "logprob": -12.6796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7890625,
+          "logprob": -12.8359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.28125,
+          "logprob": -12.296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4140625,
+          "logprob": -13.3671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.96875,
+          "logprob": -12.9765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1171875,
+          "logprob": -12.109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5390625,
+          "logprob": -12.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.609375,
+          "logprob": -12.6015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6484375,
+          "logprob": -12.625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.09375,
+          "logprob": -12.109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9921875,
+          "logprob": -13.0,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4453125,
+          "logprob": -13.4296875,
           "text": "<image>"
         },
         {
@@ -51184,17 +36390,17 @@
         },
         {
           "id": 32000,
-          "logprob": -14.7109375,
+          "logprob": -14.71875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.625,
+          "logprob": -13.6015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.765625,
+          "logprob": -13.7578125,
           "text": "<image>"
         },
         {
@@ -51204,37 +36410,37 @@
         },
         {
           "id": 32000,
-          "logprob": -14.53125,
+          "logprob": -14.5390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.34375,
+          "logprob": -13.3515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.921875,
+          "logprob": -12.9453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.359375,
+          "logprob": -11.3515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.078125,
+          "logprob": -12.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4609375,
+          "logprob": -11.421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.640625,
+          "logprob": -11.625,
           "text": "<image>"
         },
         {
@@ -51249,62 +36455,62 @@
         },
         {
           "id": 32000,
-          "logprob": -11.953125,
+          "logprob": -11.9453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0859375,
+          "logprob": -12.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.421875,
+          "logprob": -11.4296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5859375,
+          "logprob": -11.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0859375,
+          "logprob": -12.0703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.25,
+          "logprob": -11.2421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6171875,
+          "logprob": -11.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3359375,
+          "logprob": -11.3125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.46875,
+          "logprob": -11.453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.265625,
+          "logprob": -12.28125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.21875,
+          "logprob": -12.2109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1875,
+          "logprob": -12.203125,
           "text": "<image>"
         },
         {
@@ -51319,47 +36525,47 @@
         },
         {
           "id": 32000,
-          "logprob": -11.2421875,
+          "logprob": -11.25,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3046875,
+          "logprob": -12.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.03125,
+          "logprob": -11.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.203125,
+          "logprob": -11.1953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.125,
+          "logprob": -11.1171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9296875,
+          "logprob": -11.9375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9765625,
+          "logprob": -10.984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.453125,
+          "logprob": -11.390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.859375,
+          "logprob": -12.8828125,
           "text": "<image>"
         },
         {
@@ -51384,47 +36590,47 @@
         },
         {
           "id": 32000,
-          "logprob": -12.1328125,
+          "logprob": -12.1171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5,
+          "logprob": -12.515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8671875,
+          "logprob": -11.890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1640625,
+          "logprob": -13.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.765625,
+          "logprob": -15.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.609375,
+          "logprob": -11.6015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.34375,
+          "logprob": -13.328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.671875,
+          "logprob": -13.6484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.9140625,
+          "logprob": -14.875,
           "text": "<image>"
         },
         {
@@ -51434,57 +36640,87 @@
         },
         {
           "id": 32000,
-          "logprob": -13.53125,
+          "logprob": -13.5234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.25,
+          "logprob": -13.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6171875,
+          "logprob": -12.609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.8046875,
+          "logprob": -13.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.375,
+          "logprob": -14.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.171875,
+          "logprob": -12.1484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.828125,
+          "logprob": -11.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.875,
+          "logprob": -11.890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.484375,
+          "logprob": -11.4921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5625,
+          "logprob": -12.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.640625,
+          "logprob": -11.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.03125,
           "text": "<image>"
         },
         {
@@ -51492,36 +36728,6 @@
           "logprob": -11.7265625,
           "text": "<image>"
         },
-        {
-          "id": 32000,
-          "logprob": -11.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.734375,
-          "text": "<image>"
-        },
         {
           "id": 32000,
           "logprob": -11.84375,
@@ -51529,12 +36735,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0390625,
+          "logprob": -11.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.2578125,
+          "logprob": -11.2421875,
           "text": "<image>"
         },
         {
@@ -51569,22 +36775,22 @@
         },
         {
           "id": 32000,
-          "logprob": -11.0390625,
+          "logprob": -11.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1953125,
+          "logprob": -12.203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.6640625,
+          "logprob": -13.609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6640625,
+          "logprob": -12.65625,
           "text": "<image>"
         },
         {
@@ -51609,27 +36815,47 @@
         },
         {
           "id": 32000,
-          "logprob": -11.9140625,
+          "logprob": -11.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.34375,
+          "logprob": -11.3359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.578125,
+          "logprob": -11.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.953125,
+          "logprob": -11.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.296875,
+          "logprob": -11.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.875,
           "text": "<image>"
         },
         {
@@ -51639,27 +36865,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9765625,
+          "logprob": -12.046875,
           "text": "<image>"
         },
         {
@@ -51669,7 +36875,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.734375,
+          "logprob": -13.7265625,
           "text": "<image>"
         },
         {
@@ -51679,7 +36885,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.8359375,
+          "logprob": -13.8203125,
           "text": "<image>"
         },
         {
@@ -51694,37 +36900,37 @@
         },
         {
           "id": 32000,
-          "logprob": -14.5625,
+          "logprob": -14.546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.0234375,
+          "logprob": -14.015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5703125,
+          "logprob": -12.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8125,
+          "logprob": -11.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.7109375,
+          "logprob": -10.6953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6015625,
+          "logprob": -11.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8671875,
+          "logprob": -12.0234375,
           "text": "<image>"
         },
         {
@@ -51739,32 +36945,32 @@
         },
         {
           "id": 32000,
-          "logprob": -14.3359375,
+          "logprob": -14.34375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.90625,
+          "logprob": -11.8984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.296875,
+          "logprob": -13.4375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.984375,
+          "logprob": -11.9765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0625,
+          "logprob": -11.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4296875,
+          "logprob": -11.421875,
           "text": "<image>"
         },
         {
@@ -51779,27 +36985,27 @@
         },
         {
           "id": 32000,
-          "logprob": -11.28125,
+          "logprob": -11.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5546875,
+          "logprob": -11.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.671875,
+          "logprob": -11.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1171875,
+          "logprob": -12.125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5,
+          "logprob": -11.484375,
           "text": "<image>"
         },
         {
@@ -51809,27 +37015,27 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8984375,
+          "logprob": -11.875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2890625,
+          "logprob": -12.3125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6484375,
+          "logprob": -11.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8984375,
+          "logprob": -12.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.734375,
+          "logprob": -11.7421875,
           "text": "<image>"
         },
         {
@@ -51839,7 +37045,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.40625,
+          "logprob": -11.3984375,
           "text": "<image>"
         },
         {
@@ -51849,7 +37055,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.3515625,
+          "logprob": -11.34375,
           "text": "<image>"
         },
         {
@@ -51864,37 +37070,37 @@
         },
         {
           "id": 32000,
-          "logprob": -11.03125,
+          "logprob": -10.9609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.375,
+          "logprob": -13.265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6640625,
+          "logprob": -12.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.96875,
+          "logprob": -11.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.671875,
+          "logprob": -14.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.390625,
+          "logprob": -12.3828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.2890625,
+          "logprob": -14.3515625,
           "text": "<image>"
         },
         {
@@ -51904,32 +37110,32 @@
         },
         {
           "id": 32000,
-          "logprob": -15.1796875,
+          "logprob": -15.125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6875,
+          "logprob": -12.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7421875,
+          "logprob": -12.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.6484375,
+          "logprob": -14.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.546875,
+          "logprob": -13.5234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5703125,
+          "logprob": -12.5859375,
           "text": "<image>"
         },
         {
@@ -51939,32 +37145,17 @@
         },
         {
           "id": 32000,
-          "logprob": -14.390625,
+          "logprob": -14.3828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.96875,
+          "logprob": -12.9453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7421875,
+          "logprob": -12.609375,
           "text": "<image>"
         },
         {
@@ -51974,32 +37165,47 @@
         },
         {
           "id": 32000,
-          "logprob": -12.453125,
+          "logprob": -12.953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.8046875,
+          "logprob": -11.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.0078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.7890625,
+          "logprob": -12.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.3828125,
+          "logprob": -13.8828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.828125,
+          "logprob": -12.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8359375,
           "text": "<image>"
         },
         {
@@ -52009,12 +37215,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.0,
+          "logprob": -13.0078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1171875,
+          "logprob": -12.140625,
           "text": "<image>"
         },
         {
@@ -52024,17 +37230,17 @@
         },
         {
           "id": 32000,
-          "logprob": -14.015625,
+          "logprob": -13.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6328125,
+          "logprob": -11.625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4765625,
+          "logprob": -11.484375,
           "text": "<image>"
         },
         {
@@ -52044,7 +37250,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.5859375,
+          "logprob": -11.5703125,
           "text": "<image>"
         },
         {
@@ -52059,7 +37265,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0390625,
+          "logprob": -12.03125,
           "text": "<image>"
         },
         {
@@ -52069,7 +37275,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.4609375,
+          "logprob": -11.46875,
           "text": "<image>"
         },
         {
@@ -52079,7 +37285,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0625,
+          "logprob": -12.078125,
           "text": "<image>"
         },
         {
@@ -52089,142 +37295,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
+          "logprob": -11.296875,
           "text": "<image>"
         },
         {
@@ -52234,7 +37305,142 @@
         },
         {
           "id": 32000,
-          "logprob": -11.34375,
+          "logprob": -11.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.21875,
           "text": "<image>"
         },
         {
@@ -52244,52 +37450,52 @@
         },
         {
           "id": 32000,
-          "logprob": -11.3671875,
+          "logprob": -11.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0,
+          "logprob": -12.984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.1875,
+          "logprob": -15.15625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0,
+          "logprob": -11.9765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.75,
+          "logprob": -12.765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.515625,
+          "logprob": -13.4921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5703125,
+          "logprob": -11.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.703125,
+          "logprob": -11.6953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3671875,
+          "logprob": -11.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4921875,
+          "logprob": -11.5,
           "text": "<image>"
         },
         {
@@ -52299,12 +37505,12 @@
         },
         {
           "id": 32000,
-          "logprob": -14.5546875,
+          "logprob": -14.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.390625,
+          "logprob": -13.4140625,
           "text": "<image>"
         },
         {
@@ -52314,7 +37520,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.7265625,
+          "logprob": -11.71875,
           "text": "<image>"
         },
         {
@@ -52329,7 +37535,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.2421875,
+          "logprob": -13.2578125,
           "text": "<image>"
         },
         {
@@ -52349,27 +37555,27 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8203125,
+          "logprob": -11.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9296875,
+          "logprob": -11.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.796875,
+          "logprob": -11.765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1875,
+          "logprob": -11.1953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.359375,
+          "logprob": -13.3359375,
           "text": "<image>"
         },
         {
@@ -52379,7 +37585,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.6796875,
+          "logprob": -12.703125,
           "text": "<image>"
         },
         {
@@ -52394,27 +37600,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.5625,
+          "logprob": -13.6171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6484375,
+          "logprob": -13.328125,
           "text": "<image>"
         },
         {
@@ -52422,6 +37613,21 @@
           "logprob": -12.828125,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -13.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.796875,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -12.78125,
@@ -52429,42 +37635,42 @@
         },
         {
           "id": 32000,
-          "logprob": -12.578125,
+          "logprob": -12.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7578125,
+          "logprob": -12.765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.484375,
+          "logprob": -11.46875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5390625,
+          "logprob": -12.484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6015625,
+          "logprob": -11.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.640625,
+          "logprob": -11.6171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.140625,
+          "logprob": -12.1328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.65625,
+          "logprob": -12.59375,
           "text": "<image>"
         },
         {
@@ -52474,17 +37680,17 @@
         },
         {
           "id": 32000,
-          "logprob": -14.265625,
+          "logprob": -14.25,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0078125,
+          "logprob": -12.0234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7578125,
+          "logprob": -11.7734375,
           "text": "<image>"
         },
         {
@@ -52499,52 +37705,52 @@
         },
         {
           "id": 32000,
-          "logprob": -11.625,
+          "logprob": -11.609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.53125,
+          "logprob": -13.5234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.046875,
+          "logprob": -12.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6796875,
+          "logprob": -12.6953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7109375,
+          "logprob": -11.703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.890625,
+          "logprob": -11.8828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8359375,
+          "logprob": -11.84375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7734375,
+          "logprob": -11.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3671875,
+          "logprob": -12.3515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.3125,
+          "logprob": -14.3046875,
           "text": "<image>"
         },
         {
@@ -52554,22 +37760,22 @@
         },
         {
           "id": 32000,
-          "logprob": -11.921875,
+          "logprob": -11.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3203125,
+          "logprob": -11.328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1640625,
+          "logprob": -14.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.2421875,
+          "logprob": -11.25,
           "text": "<image>"
         },
         {
@@ -52579,7 +37785,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.078125,
           "text": "<image>"
         },
         {
@@ -52589,12 +37795,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.2578125,
+          "logprob": -12.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.7265625,
+          "logprob": -13.71875,
           "text": "<image>"
         },
         {
@@ -52614,12 +37820,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.3828125,
+          "logprob": -13.390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5390625,
+          "logprob": -12.515625,
           "text": "<image>"
         },
         {
@@ -52629,22 +37835,22 @@
         },
         {
           "id": 32000,
-          "logprob": -14.15625,
+          "logprob": -14.1796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5703125,
+          "logprob": -11.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5546875,
+          "logprob": -11.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.65625,
+          "logprob": -11.671875,
           "text": "<image>"
         },
         {
@@ -52654,47 +37860,47 @@
         },
         {
           "id": 32000,
-          "logprob": -12.6484375,
+          "logprob": -12.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5390625,
+          "logprob": -12.515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7734375,
+          "logprob": -11.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0859375,
+          "logprob": -13.1328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9140625,
+          "logprob": -12.8984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.6015625,
+          "logprob": -14.609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5390625,
+          "logprob": -11.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8828125,
+          "logprob": -11.8515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.078125,
+          "logprob": -13.0625,
           "text": "<image>"
         },
         {
@@ -52704,17 +37910,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.15625,
+          "logprob": -12.1328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.03125,
+          "logprob": -12.0234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4453125,
+          "logprob": -12.453125,
           "text": "<image>"
         },
         {
@@ -52724,7 +37930,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.9375,
+          "logprob": -11.9296875,
           "text": "<image>"
         },
         {
@@ -52739,42 +37945,42 @@
         },
         {
           "id": 32000,
-          "logprob": -12.7890625,
+          "logprob": -12.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0390625,
+          "logprob": -13.0234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8203125,
+          "logprob": -12.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.578125,
+          "logprob": -12.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.7578125,
+          "logprob": -10.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9609375,
+          "logprob": -11.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.484375,
+          "logprob": -11.4921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.265625,
+          "logprob": -11.25,
           "text": "<image>"
         },
         {
@@ -52784,17 +37990,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.71875,
+          "logprob": -12.6953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7890625,
+          "logprob": -11.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5234375,
+          "logprob": -11.515625,
           "text": "<image>"
         },
         {
@@ -52804,7 +38010,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.46875,
+          "logprob": -12.4765625,
           "text": "<image>"
         },
         {
@@ -52819,17 +38025,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.65625,
+          "logprob": -11.6484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5234375,
+          "logprob": -11.515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7890625,
+          "logprob": -12.8125,
           "text": "<image>"
         },
         {
@@ -52839,17 +38045,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.28125,
+          "logprob": -13.2890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.109375,
+          "logprob": -11.09375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.34375,
+          "logprob": -12.3359375,
           "text": "<image>"
         },
         {
@@ -52859,27 +38065,27 @@
         },
         {
           "id": 32000,
-          "logprob": -13.0,
+          "logprob": -13.0078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.28125,
+          "logprob": -15.328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.8046875,
+          "logprob": -14.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0234375,
+          "logprob": -11.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.109375,
+          "logprob": -13.0703125,
           "text": "<image>"
         },
         {
@@ -52889,12 +38095,12 @@
         },
         {
           "id": 32000,
-          "logprob": -14.0078125,
+          "logprob": -14.0859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.7109375,
+          "logprob": -14.734375,
           "text": "<image>"
         },
         {
@@ -52909,127 +38115,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.9765625,
+          "logprob": -13.953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.3046875,
+          "logprob": -14.28125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.78125,
+          "logprob": -14.8828125,
           "text": "<image>"
         },
         {
@@ -53039,22 +38135,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.859375,
+          "logprob": -12.28125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.765625,
+          "logprob": -12.34375,
           "text": "<image>"
         },
         {
@@ -53069,117 +38155,87 @@
         },
         {
           "id": 32000,
-          "logprob": -11.71875,
+          "logprob": -12.4296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.25,
+          "logprob": -11.8984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.65625,
+          "logprob": -11.984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.84375,
+          "logprob": -11.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0390625,
+          "logprob": -11.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4921875,
+          "logprob": -12.1875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8828125,
+          "logprob": -11.6171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1015625,
+          "logprob": -12.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.0625,
+          "logprob": -12.4765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.09375,
+          "logprob": -12.953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2578125,
+          "logprob": -10.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.4453125,
+          "logprob": -13.1015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9921875,
+          "logprob": -12.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.140625,
+          "logprob": -11.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.828125,
+          "logprob": -11.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5546875,
+          "logprob": -11.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6015625,
+          "logprob": -11.765625,
           "text": "<image>"
         },
         {
@@ -53189,32 +38245,182 @@
         },
         {
           "id": 32000,
-          "logprob": -12.84375,
+          "logprob": -12.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.78125,
+          "logprob": -11.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1171875,
+          "logprob": -12.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8046875,
+          "logprob": -11.71875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5234375,
+          "logprob": -11.34375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4609375,
+          "logprob": -12.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4765625,
           "text": "<image>"
         },
         {
@@ -53234,22 +38440,22 @@
         },
         {
           "id": 32000,
-          "logprob": -12.953125,
+          "logprob": -12.9609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.859375,
+          "logprob": -12.8515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0390625,
+          "logprob": -12.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.28125,
+          "logprob": -13.234375,
           "text": "<image>"
         },
         {
@@ -53259,12 +38465,12 @@
         },
         {
           "id": 32000,
-          "logprob": -10.8203125,
+          "logprob": -10.8515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.6953125,
+          "logprob": -10.671875,
           "text": "<image>"
         },
         {
@@ -53274,7 +38480,7 @@
         },
         {
           "id": 32000,
-          "logprob": -10.9375,
+          "logprob": -10.9140625,
           "text": "<image>"
         },
         {
@@ -53294,17 +38500,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.3046875,
+          "logprob": -12.2890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9609375,
+          "logprob": -10.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6328125,
+          "logprob": -11.6015625,
           "text": "<image>"
         },
         {
@@ -53314,17 +38520,17 @@
         },
         {
           "id": 32000,
-          "logprob": -10.84375,
+          "logprob": -10.8515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.140625,
+          "logprob": -11.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.09375,
+          "logprob": -11.0859375,
           "text": "<image>"
         },
         {
@@ -53334,12 +38540,12 @@
         },
         {
           "id": 32000,
-          "logprob": -15.578125,
+          "logprob": -15.765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.875,
+          "logprob": -11.8515625,
           "text": "<image>"
         },
         {
@@ -53354,22 +38560,22 @@
         },
         {
           "id": 32000,
-          "logprob": -10.984375,
+          "logprob": -10.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.234375,
+          "logprob": -12.25,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8359375,
+          "logprob": -11.828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8515625,
+          "logprob": -11.84375,
           "text": "<image>"
         },
         {
@@ -53384,7 +38590,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.5625,
+          "logprob": -13.5703125,
           "text": "<image>"
         },
         {
@@ -53394,37 +38600,37 @@
         },
         {
           "id": 32000,
-          "logprob": -13.5859375,
+          "logprob": -13.6171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.0234375,
+          "logprob": -14.0078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1796875,
+          "logprob": -11.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1484375,
+          "logprob": -14.140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.234375,
+          "logprob": -12.25,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.734375,
+          "logprob": -11.7265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3515625,
+          "logprob": -12.3203125,
           "text": "<image>"
         },
         {
@@ -53434,17 +38640,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.90625,
+          "logprob": -11.8828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3359375,
+          "logprob": -12.3125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5703125,
+          "logprob": -11.5625,
           "text": "<image>"
         },
         {
@@ -53454,17 +38660,17 @@
         },
         {
           "id": 32000,
-          "logprob": -14.6875,
+          "logprob": -14.6796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0625,
+          "logprob": -11.046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5234375,
+          "logprob": -11.5,
           "text": "<image>"
         },
         {
@@ -53489,12 +38695,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.46875,
+          "logprob": -12.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.0859375,
           "text": "<image>"
         },
         {
@@ -53507,21 +38713,6 @@
           "logprob": -12.46875,
           "text": "<image>"
         },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6640625,
-          "text": "<image>"
-        },
         {
           "id": 32000,
           "logprob": -11.9765625,
@@ -53529,7 +38720,22 @@
         },
         {
           "id": 32000,
-          "logprob": -10.8671875,
+          "logprob": -11.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.875,
           "text": "<image>"
         },
         {
@@ -53549,12 +38755,12 @@
         },
         {
           "id": 32000,
-          "logprob": -10.796875,
+          "logprob": -10.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.59375,
+          "logprob": -11.6171875,
           "text": "<image>"
         },
         {
@@ -53584,22 +38790,22 @@
         },
         {
           "id": 32000,
-          "logprob": -11.390625,
+          "logprob": -11.3828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9375,
+          "logprob": -10.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.6796875,
+          "logprob": -10.703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9375,
+          "logprob": -11.90625,
           "text": "<image>"
         },
         {
@@ -53609,82 +38815,57 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8125,
+          "logprob": -11.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3671875,
+          "logprob": -11.4453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -9.625,
+          "logprob": -9.6328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.8515625,
+          "logprob": -10.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9453125,
+          "logprob": -13.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8203125,
+          "logprob": -12.890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3359375,
+          "logprob": -13.4765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.015625,
+          "logprob": -14.109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8671875,
+          "logprob": -11.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4453125,
+          "logprob": -13.484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.75,
+          "logprob": -13.5625,
           "text": "<image>"
         },
         {
@@ -53692,6 +38873,31 @@
           "logprob": -12.859375,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -10.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.890625,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -13.0625,
@@ -53699,7 +38905,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.21875,
+          "logprob": -11.2265625,
           "text": "<image>"
         },
         {
@@ -53709,7 +38915,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.296875,
+          "logprob": -12.2890625,
           "text": "<image>"
         },
         {
@@ -53719,12 +38925,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.8671875,
+          "logprob": -12.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.6953125,
+          "logprob": -15.671875,
           "text": "<image>"
         },
         {
@@ -53739,7 +38945,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.140625,
+          "logprob": -12.1328125,
           "text": "<image>"
         },
         {
@@ -53749,22 +38955,22 @@
         },
         {
           "id": 32000,
-          "logprob": -11.390625,
+          "logprob": -11.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.328125,
+          "logprob": -12.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.265625,
+          "logprob": -12.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9453125,
+          "logprob": -10.9296875,
           "text": "<image>"
         },
         {
@@ -53779,7 +38985,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.6015625,
+          "logprob": -11.59375,
           "text": "<image>"
         },
         {
@@ -53789,7 +38995,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.546875,
+          "logprob": -11.53125,
           "text": "<image>"
         },
         {
@@ -53804,7 +39010,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.6796875,
+          "logprob": -11.6875,
           "text": "<image>"
         },
         {
@@ -53819,27 +39025,27 @@
         },
         {
           "id": 32000,
-          "logprob": -12.125,
+          "logprob": -12.1171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5703125,
+          "logprob": -12.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.109375,
+          "logprob": -12.1015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0078125,
+          "logprob": -12.0,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.734375,
+          "logprob": -13.7265625,
           "text": "<image>"
         },
         {
@@ -53849,67 +39055,67 @@
         },
         {
           "id": 32000,
-          "logprob": -11.6484375,
+          "logprob": -11.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.34375,
+          "logprob": -12.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.140625,
+          "logprob": -12.1484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1796875,
+          "logprob": -14.21875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.8046875,
+          "logprob": -14.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -18.046875,
+          "logprob": -17.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.34375,
+          "logprob": -15.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.4453125,
+          "logprob": -14.4765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.625,
+          "logprob": -10.6015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0546875,
+          "logprob": -12.0703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.859375,
+          "logprob": -12.84375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2265625,
+          "logprob": -12.2421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2109375,
+          "logprob": -13.203125,
           "text": "<image>"
         },
         {
@@ -53919,37 +39125,37 @@
         },
         {
           "id": 32000,
-          "logprob": -10.9296875,
+          "logprob": -10.9375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9140625,
+          "logprob": -11.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.125,
+          "logprob": -12.1015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9609375,
+          "logprob": -10.9453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7734375,
+          "logprob": -11.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.828125,
+          "logprob": -11.8203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2890625,
+          "logprob": -12.296875,
           "text": "<image>"
         },
         {
@@ -53959,27 +39165,27 @@
         },
         {
           "id": 32000,
-          "logprob": -11.53125,
+          "logprob": -11.5390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0234375,
+          "logprob": -12.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.2421875,
+          "logprob": -15.2265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1171875,
+          "logprob": -12.125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1171875,
+          "logprob": -12.125,
           "text": "<image>"
         },
         {
@@ -53989,7 +39195,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.671875,
+          "logprob": -11.6796875,
           "text": "<image>"
         },
         {
@@ -54009,32 +39215,32 @@
         },
         {
           "id": 32000,
-          "logprob": -12.734375,
+          "logprob": -12.703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.609375,
+          "logprob": -11.6328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5234375,
+          "logprob": -11.515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5078125,
+          "logprob": -11.546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2109375,
+          "logprob": -12.21875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.421875,
+          "logprob": -11.4140625,
           "text": "<image>"
         },
         {
@@ -54044,137 +39250,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.96875,
+          "logprob": -11.9765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.46875,
+          "logprob": -12.9609375,
           "text": "<image>"
         },
         {
@@ -54182,6 +39263,131 @@
           "logprob": -12.578125,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -11.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5078125,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -11.6171875,
@@ -54189,32 +39395,32 @@
         },
         {
           "id": 32000,
-          "logprob": -13.8125,
+          "logprob": -13.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1640625,
+          "logprob": -11.203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.765625,
+          "logprob": -11.78125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.5,
+          "logprob": -16.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3046875,
+          "logprob": -12.296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1171875,
+          "logprob": -14.1484375,
           "text": "<image>"
         },
         {
@@ -54224,12 +39430,47 @@
         },
         {
           "id": 32000,
-          "logprob": -12.5,
+          "logprob": -12.4921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.6015625,
+          "logprob": -10.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1953125,
           "text": "<image>"
         },
         {
@@ -54239,52 +39480,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.625,
+          "logprob": -11.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3046875,
+          "logprob": -12.6484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1171875,
+          "logprob": -11.109375,
           "text": "<image>"
         },
         {
@@ -54294,7 +39500,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.921875,
+          "logprob": -13.90625,
           "text": "<image>"
         },
         {
@@ -54309,12 +39515,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.3671875,
+          "logprob": -13.375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.7109375,
+          "logprob": -14.703125,
           "text": "<image>"
         },
         {
@@ -54324,7 +39530,7 @@
         },
         {
           "id": 32000,
-          "logprob": -14.859375,
+          "logprob": -14.890625,
           "text": "<image>"
         },
         {
@@ -54339,37 +39545,37 @@
         },
         {
           "id": 32000,
-          "logprob": -11.9296875,
+          "logprob": -11.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3828125,
+          "logprob": -13.3984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.046875,
+          "logprob": -13.9453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1484375,
+          "logprob": -13.1875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.265625,
+          "logprob": -11.28125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.125,
+          "logprob": -15.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.046875,
+          "logprob": -15.0234375,
           "text": "<image>"
         },
         {
@@ -54384,92 +39590,22 @@
         },
         {
           "id": 32000,
-          "logprob": -14.09375,
+          "logprob": -14.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.890625,
+          "logprob": -13.8984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.3203125,
+          "logprob": -15.3125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9609375,
+          "logprob": -12.8125,
           "text": "<image>"
         },
         {
@@ -54479,22 +39615,92 @@
         },
         {
           "id": 32000,
-          "logprob": -12.3203125,
+          "logprob": -13.046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.59375,
+          "logprob": -10.625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.171875,
+          "logprob": -11.25,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5,
+          "logprob": -12.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4921875,
           "text": "<image>"
         },
         {
@@ -54524,17 +39730,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.78125,
+          "logprob": -12.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3125,
+          "logprob": -11.3046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1171875,
+          "logprob": -11.109375,
           "text": "<image>"
         },
         {
@@ -54544,27 +39750,27 @@
         },
         {
           "id": 32000,
-          "logprob": -13.0859375,
+          "logprob": -13.0234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.921875,
+          "logprob": -10.9296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.515625,
+          "logprob": -13.5078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4921875,
+          "logprob": -11.5,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8203125,
+          "logprob": -11.8359375,
           "text": "<image>"
         },
         {
@@ -54574,42 +39780,42 @@
         },
         {
           "id": 32000,
-          "logprob": -14.0078125,
+          "logprob": -14.0234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.21875,
+          "logprob": -15.2578125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -17.84375,
+          "logprob": -17.953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.046875,
+          "logprob": -12.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.296875,
+          "logprob": -12.28125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7109375,
+          "logprob": -11.6875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1015625,
+          "logprob": -12.1171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.515625,
+          "logprob": -11.5546875,
           "text": "<image>"
         },
         {
@@ -54619,17 +39825,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.640625,
+          "logprob": -12.6171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4609375,
+          "logprob": -13.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.84375,
+          "logprob": -14.828125,
           "text": "<image>"
         },
         {
@@ -54639,22 +39845,22 @@
         },
         {
           "id": 32000,
-          "logprob": -11.6171875,
+          "logprob": -11.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.765625,
+          "logprob": -12.7421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5390625,
+          "logprob": -13.484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4453125,
+          "logprob": -11.4609375,
           "text": "<image>"
         },
         {
@@ -54664,37 +39870,37 @@
         },
         {
           "id": 32000,
-          "logprob": -11.96875,
+          "logprob": -11.984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4765625,
+          "logprob": -12.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.890625,
+          "logprob": -15.9375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.953125,
+          "logprob": -11.984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.40625,
+          "logprob": -11.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0,
+          "logprob": -12.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1875,
+          "logprob": -14.3046875,
           "text": "<image>"
         },
         {
@@ -54704,7 +39910,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.125,
+          "logprob": -12.1171875,
           "text": "<image>"
         },
         {
@@ -54714,22 +39920,22 @@
         },
         {
           "id": 32000,
-          "logprob": -13.140625,
+          "logprob": -13.1484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1484375,
+          "logprob": -12.1875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.375,
+          "logprob": -13.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.9453125,
+          "logprob": -13.9921875,
           "text": "<image>"
         },
         {
@@ -54739,12 +39945,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0859375,
+          "logprob": -12.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.734375,
+          "logprob": -10.7421875,
           "text": "<image>"
         },
         {
@@ -54754,12 +39960,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.65625,
+          "logprob": -12.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.953125,
+          "logprob": -11.9765625,
           "text": "<image>"
         },
         {
@@ -54769,12 +39975,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.5,
+          "logprob": -12.4765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8984375,
+          "logprob": -11.890625,
           "text": "<image>"
         },
         {
@@ -54794,22 +40000,22 @@
         },
         {
           "id": 32000,
-          "logprob": -12.4375,
+          "logprob": -12.4609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5703125,
+          "logprob": -12.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.828125,
+          "logprob": -12.8203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.375,
+          "logprob": -13.359375,
           "text": "<image>"
         },
         {
@@ -54829,7 +40035,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.2421875,
+          "logprob": -11.234375,
           "text": "<image>"
         },
         {
@@ -54839,27 +40045,27 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.328125,
+          "logprob": -15.3046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.890625,
+          "logprob": -12.8828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4453125,
+          "logprob": -13.453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8203125,
+          "logprob": -11.8125,
           "text": "<image>"
         },
         {
@@ -54869,17 +40075,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.96875,
+          "logprob": -12.9765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9921875,
+          "logprob": -11.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.421875,
+          "logprob": -13.4296875,
           "text": "<image>"
         },
         {
@@ -54889,12 +40095,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.265625,
+          "logprob": -13.28125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.265625,
+          "logprob": -13.2578125,
           "text": "<image>"
         },
         {
@@ -54909,12 +40115,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.15625,
+          "logprob": -12.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.375,
+          "logprob": -10.3828125,
           "text": "<image>"
         },
         {
@@ -54924,7 +40130,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.2109375,
+          "logprob": -13.203125,
           "text": "<image>"
         },
         {
@@ -54934,7 +40140,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.15625,
+          "logprob": -12.1328125,
           "text": "<image>"
         },
         {
@@ -54944,12 +40150,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.171875,
+          "logprob": -13.1796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6328125,
+          "logprob": -11.65625,
           "text": "<image>"
         },
         {
@@ -54959,12 +40165,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.90625,
+          "logprob": -11.9375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.765625,
+          "logprob": -11.7734375,
           "text": "<image>"
         },
         {
@@ -54979,7 +40185,7 @@
         },
         {
           "id": 32000,
-          "logprob": -10.8984375,
+          "logprob": -10.90625,
           "text": "<image>"
         },
         {
@@ -54989,17 +40195,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.40625,
+          "logprob": -12.3828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5390625,
+          "logprob": -11.5234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.59375,
+          "logprob": -16.578125,
           "text": "<image>"
         },
         {
@@ -55009,7 +40215,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.9140625,
+          "logprob": -11.9453125,
           "text": "<image>"
         },
         {
@@ -55024,22 +40230,22 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8984375,
+          "logprob": -11.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0703125,
+          "logprob": -13.0625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.03125,
+          "logprob": -12.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.921875,
+          "logprob": -11.9296875,
           "text": "<image>"
         },
         {
@@ -55049,12 +40255,12 @@
         },
         {
           "id": 32000,
-          "logprob": -10.5390625,
+          "logprob": -10.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.453125,
+          "logprob": -11.4375,
           "text": "<image>"
         },
         {
@@ -55064,22 +40270,22 @@
         },
         {
           "id": 32000,
-          "logprob": -13.0859375,
+          "logprob": -13.09375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1796875,
+          "logprob": -12.15625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.8203125,
+          "logprob": -15.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.984375,
+          "logprob": -10.9765625,
           "text": "<image>"
         },
         {
@@ -55089,42 +40295,42 @@
         },
         {
           "id": 32000,
-          "logprob": -10.109375,
+          "logprob": -10.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8671875,
+          "logprob": -11.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1328125,
+          "logprob": -13.125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.640625,
+          "logprob": -12.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.015625,
+          "logprob": -16.046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5078125,
+          "logprob": -11.515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.7265625,
+          "logprob": -13.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.703125,
+          "logprob": -14.671875,
           "text": "<image>"
         },
         {
@@ -55134,32 +40340,32 @@
         },
         {
           "id": 32000,
-          "logprob": -12.2421875,
+          "logprob": -12.2109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.6640625,
+          "logprob": -10.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.6640625,
+          "logprob": -15.6484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9140625,
+          "logprob": -10.9296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3203125,
+          "logprob": -11.328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.046875,
+          "logprob": -11.0546875,
           "text": "<image>"
         },
         {
@@ -55169,7 +40375,7 @@
         },
         {
           "id": 32000,
-          "logprob": -10.953125,
+          "logprob": -10.96875,
           "text": "<image>"
         },
         {
@@ -55184,337 +40390,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2578125,
+          "logprob": -12.359375,
           "text": "<image>"
         },
         {
@@ -55524,22 +40400,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.84375,
+          "logprob": -14.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
+          "logprob": -11.4921875,
           "text": "<image>"
         },
         {
@@ -55549,87 +40415,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.125,
+          "logprob": -12.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.390625,
+          "logprob": -12.640625,
           "text": "<image>"
         },
         {
@@ -55639,87 +40430,297 @@
         },
         {
           "id": 32000,
-          "logprob": -12.5078125,
+          "logprob": -12.4375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.09375,
+          "logprob": -11.8828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.8515625,
+          "logprob": -12.15625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8203125,
+          "logprob": -11.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.734375,
+          "logprob": -13.0,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2578125,
+          "logprob": -12.0703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.28125,
+          "logprob": -11.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.859375,
+          "logprob": -13.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.703125,
+          "logprob": -12.1484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5703125,
+          "logprob": -13.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.59375,
+          "logprob": -12.203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.890625,
+          "logprob": -11.9375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3359375,
+          "logprob": -12.1796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0625,
+          "logprob": -10.7421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.53125,
+          "logprob": -10.8203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7109375,
+          "logprob": -12.375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.7578125,
+          "logprob": -11.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.25,
           "text": "<image>"
         },
         {
@@ -55729,17 +40730,47 @@
         },
         {
           "id": 32000,
-          "logprob": -10.75,
+          "logprob": -11.8515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.2265625,
+          "logprob": -13.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9296875,
+          "logprob": -10.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1328125,
           "text": "<image>"
         },
         {
@@ -55749,7 +40780,12 @@
         },
         {
           "id": 32000,
-          "logprob": -10.5390625,
+          "logprob": -14.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0546875,
           "text": "<image>"
         },
         {
@@ -55757,6 +40793,56 @@
           "logprob": -11.6796875,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -11.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.234375,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -12.5390625,
@@ -55764,7 +40850,127 @@
         },
         {
           "id": 32000,
-          "logprob": -16.953125,
+          "logprob": -15.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.0,
           "text": "<image>"
         },
         {
@@ -55774,12 +40980,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.3359375,
+          "logprob": -11.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.671875,
+          "logprob": -11.6796875,
           "text": "<image>"
         },
         {
@@ -55794,12 +41000,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8671875,
+          "logprob": -11.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.484375,
+          "logprob": -11.4765625,
           "text": "<image>"
         },
         {
@@ -55809,12 +41015,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.6640625,
+          "logprob": -12.6875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4296875,
+          "logprob": -13.421875,
           "text": "<image>"
         },
         {
@@ -55829,7 +41035,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.3046875,
+          "logprob": -12.3125,
           "text": "<image>"
         },
         {
@@ -55839,17 +41045,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.34375,
+          "logprob": -13.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0703125,
+          "logprob": -13.0625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.234375,
+          "logprob": -11.2421875,
           "text": "<image>"
         },
         {
@@ -55859,37 +41065,37 @@
         },
         {
           "id": 32000,
-          "logprob": -13.4921875,
+          "logprob": -13.5,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.7578125,
+          "logprob": -10.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.53125,
+          "logprob": -13.5390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6953125,
+          "logprob": -12.703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.203125,
+          "logprob": -13.1953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -17.078125,
+          "logprob": -17.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.25,
+          "logprob": -15.0234375,
           "text": "<image>"
         },
         {
@@ -55899,17 +41105,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.140625,
+          "logprob": -12.125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5234375,
+          "logprob": -11.515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.0625,
+          "logprob": -16.125,
           "text": "<image>"
         },
         {
@@ -55919,17 +41125,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.6015625,
+          "logprob": -11.609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.5859375,
+          "logprob": -14.6015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.890625,
+          "logprob": -11.8984375,
           "text": "<image>"
         },
         {
@@ -55944,22 +41150,22 @@
         },
         {
           "id": 32000,
-          "logprob": -11.7734375,
+          "logprob": -11.78125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9921875,
+          "logprob": -13.0,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9375,
+          "logprob": -12.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9921875,
+          "logprob": -12.0,
           "text": "<image>"
         },
         {
@@ -55969,7 +41175,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.7578125,
+          "logprob": -11.7734375,
           "text": "<image>"
         },
         {
@@ -55979,7 +41185,7 @@
         },
         {
           "id": 32000,
-          "logprob": -14.328125,
+          "logprob": -14.3203125,
           "text": "<image>"
         },
         {
@@ -55989,22 +41195,22 @@
         },
         {
           "id": 32000,
-          "logprob": -13.84375,
+          "logprob": -13.875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.2109375,
+          "logprob": -14.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.390625,
+          "logprob": -12.375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.953125,
+          "logprob": -13.9375,
           "text": "<image>"
         },
         {
@@ -56014,27 +41220,27 @@
         },
         {
           "id": 32000,
-          "logprob": -14.6796875,
+          "logprob": -14.609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.84375,
+          "logprob": -11.8359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.65625,
+          "logprob": -10.6484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0078125,
+          "logprob": -12.0,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0859375,
+          "logprob": -11.0703125,
           "text": "<image>"
         },
         {
@@ -56054,7 +41260,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.6875,
+          "logprob": -12.6640625,
           "text": "<image>"
         },
         {
@@ -56069,22 +41275,22 @@
         },
         {
           "id": 32000,
-          "logprob": -16.109375,
+          "logprob": -16.09375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.984375,
+          "logprob": -11.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.40625,
+          "logprob": -13.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.375,
+          "logprob": -13.453125,
           "text": "<image>"
         },
         {
@@ -56094,57 +41300,57 @@
         },
         {
           "id": 32000,
-          "logprob": -12.921875,
+          "logprob": -12.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.859375,
+          "logprob": -14.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3671875,
+          "logprob": -13.375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0546875,
+          "logprob": -12.0703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.21875,
+          "logprob": -15.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6953125,
+          "logprob": -11.6875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.5234375,
+          "logprob": -10.5390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.8984375,
+          "logprob": -14.703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.8515625,
+          "logprob": -13.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6796875,
+          "logprob": -11.6875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -17.4375,
+          "logprob": -17.453125,
           "text": "<image>"
         },
         {
@@ -56159,7 +41365,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8125,
+          "logprob": -11.7890625,
           "text": "<image>"
         },
         {
@@ -56174,12 +41380,12 @@
         },
         {
           "id": 32000,
-          "logprob": -14.2421875,
+          "logprob": -14.265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.7265625,
+          "logprob": -13.6953125,
           "text": "<image>"
         },
         {
@@ -56194,57 +41400,57 @@
         },
         {
           "id": 32000,
-          "logprob": -12.84375,
+          "logprob": -12.890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9140625,
+          "logprob": -11.9375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.7421875,
+          "logprob": -14.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.984375,
+          "logprob": -11.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.71875,
+          "logprob": -14.7265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.578125,
+          "logprob": -11.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.015625,
+          "logprob": -13.0,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.796875,
+          "logprob": -12.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0078125,
+          "logprob": -11.015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.46875,
+          "logprob": -16.421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9296875,
+          "logprob": -11.921875,
           "text": "<image>"
         },
         {
@@ -56254,7 +41460,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.8515625,
+          "logprob": -12.859375,
           "text": "<image>"
         },
         {
@@ -56269,12 +41475,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.140625,
+          "logprob": -12.1328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.359375,
+          "logprob": -13.3515625,
           "text": "<image>"
         },
         {
@@ -56289,27 +41495,27 @@
         },
         {
           "id": 32000,
-          "logprob": -11.578125,
+          "logprob": -11.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8203125,
+          "logprob": -12.84375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.984375,
+          "logprob": -11.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5078125,
+          "logprob": -12.515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.625,
+          "logprob": -12.6328125,
           "text": "<image>"
         },
         {
@@ -56324,7 +41530,7 @@
         },
         {
           "id": 32000,
-          "logprob": -14.0546875,
+          "logprob": -14.046875,
           "text": "<image>"
         },
         {
@@ -56334,7 +41540,7 @@
         },
         {
           "id": 32000,
-          "logprob": -17.21875,
+          "logprob": -17.203125,
           "text": "<image>"
         },
         {
@@ -56344,17 +41550,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.3671875,
+          "logprob": -13.375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.6171875,
+          "logprob": -14.609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9609375,
+          "logprob": -12.953125,
           "text": "<image>"
         },
         {
@@ -56364,12 +41570,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.7265625,
+          "logprob": -11.71875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.03125,
+          "logprob": -15.0234375,
           "text": "<image>"
         },
         {
@@ -56379,7 +41585,7 @@
         },
         {
           "id": 32000,
-          "logprob": -15.890625,
+          "logprob": -15.9296875,
           "text": "<image>"
         },
         {
@@ -56404,22 +41610,22 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0234375,
+          "logprob": -12.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7578125,
+          "logprob": -12.75,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.7578125,
+          "logprob": -14.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5,
+          "logprob": -11.484375,
           "text": "<image>"
         },
         {
@@ -56429,7 +41635,7 @@
         },
         {
           "id": 32000,
-          "logprob": -10.7265625,
+          "logprob": -10.734375,
           "text": "<image>"
         },
         {
@@ -56439,22 +41645,22 @@
         },
         {
           "id": 32000,
-          "logprob": -11.4140625,
+          "logprob": -11.421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9453125,
+          "logprob": -12.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1171875,
+          "logprob": -13.0625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0390625,
+          "logprob": -12.03125,
           "text": "<image>"
         },
         {
@@ -56464,7 +41670,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.234375,
+          "logprob": -12.2109375,
           "text": "<image>"
         },
         {
@@ -56479,7 +41685,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.0390625,
+          "logprob": -13.09375,
           "text": "<image>"
         },
         {
@@ -56489,7 +41695,7 @@
         },
         {
           "id": 32000,
-          "logprob": -15.265625,
+          "logprob": -15.2578125,
           "text": "<image>"
         },
         {
@@ -56504,7 +41710,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.453125,
+          "logprob": -11.4609375,
           "text": "<image>"
         },
         {
@@ -56519,12 +41725,12 @@
         },
         {
           "id": 32000,
-          "logprob": -10.6796875,
+          "logprob": -10.6875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.0546875,
           "text": "<image>"
         },
         {
@@ -56534,7 +41740,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.421875,
+          "logprob": -12.4140625,
           "text": "<image>"
         },
         {
@@ -56544,27 +41750,27 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8046875,
+          "logprob": -11.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.3828125,
+          "logprob": -14.4609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.734375,
+          "logprob": -11.7109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.890625,
+          "logprob": -12.828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.859375,
+          "logprob": -12.8359375,
           "text": "<image>"
         },
         {
@@ -56574,27 +41780,27 @@
         },
         {
           "id": 32000,
-          "logprob": -13.5625,
+          "logprob": -13.5390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.25,
+          "logprob": -16.21875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.8125,
+          "logprob": -16.84375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.921875,
+          "logprob": -11.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.375,
+          "logprob": -14.359375,
           "text": "<image>"
         },
         {
@@ -56604,12 +41810,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.671875,
+          "logprob": -13.6953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9921875,
+          "logprob": -12.984375,
           "text": "<image>"
         },
         {
@@ -56624,57 +41830,37 @@
         },
         {
           "id": 32000,
-          "logprob": -17.5625,
+          "logprob": -17.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.625,
+          "logprob": -12.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.0625,
+          "logprob": -15.046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.75,
+          "logprob": -16.71875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.84375,
+          "logprob": -11.8359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.328125,
+          "logprob": -13.3515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
+          "logprob": -12.2734375,
           "text": "<image>"
         },
         {
@@ -56684,17 +41870,37 @@
         },
         {
           "id": 32000,
-          "logprob": -13.109375,
+          "logprob": -11.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.625,
+          "logprob": -12.46875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.890625,
+          "logprob": -11.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.921875,
           "text": "<image>"
         },
         {
@@ -56704,7 +41910,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.15625,
+          "logprob": -13.21875,
           "text": "<image>"
         },
         {
@@ -56724,7 +41930,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.5390625,
+          "logprob": -12.5625,
           "text": "<image>"
         },
         {
@@ -56734,12 +41940,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.9609375,
+          "logprob": -11.9453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.828125,
+          "logprob": -11.8359375,
           "text": "<image>"
         },
         {
@@ -56749,22 +41955,22 @@
         },
         {
           "id": 32000,
-          "logprob": -12.8125,
+          "logprob": -12.875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.96875,
+          "logprob": -11.9609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.109375,
+          "logprob": -13.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6328125,
+          "logprob": -12.625,
           "text": "<image>"
         },
         {
@@ -56784,12 +41990,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.109375,
+          "logprob": -12.0859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.265625,
+          "logprob": -14.25,
           "text": "<image>"
         },
         {
@@ -56799,127 +42005,17 @@
         },
         {
           "id": 32000,
-          "logprob": -16.640625,
+          "logprob": -16.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.828125,
+          "logprob": -12.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -19.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5859375,
+          "logprob": -12.7890625,
           "text": "<image>"
         },
         {
@@ -56929,852 +42025,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.3046875,
+          "logprob": -12.3125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5234375,
+          "logprob": -12.3203125,
           "text": "<image>"
         },
         {
@@ -57784,107 +42040,137 @@
         },
         {
           "id": 32000,
-          "logprob": -11.6875,
+          "logprob": -14.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.765625,
+          "logprob": -14.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4140625,
+          "logprob": -12.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -18.703125,
+          "logprob": -14.546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.6484375,
+          "logprob": -19.4375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.2109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.703125,
+          "logprob": -13.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.0859375,
+          "logprob": -11.8515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.0703125,
+          "logprob": -11.4453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.5234375,
+          "logprob": -12.2265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3671875,
+          "logprob": -12.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0390625,
+          "logprob": -12.3359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.640625,
+          "logprob": -17.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8359375,
+          "logprob": -17.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1015625,
+          "logprob": -11.515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3515625,
+          "logprob": -12.0078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.578125,
+          "logprob": -13.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.25,
+          "logprob": -12.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.15625,
+          "logprob": -12.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7265625,
+          "logprob": -12.328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.90625,
+          "logprob": -13.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.5,
           "text": "<image>"
         },
         {
@@ -57899,17 +42185,32 @@
         },
         {
           "id": 32000,
-          "logprob": -10.5625,
+          "logprob": -14.875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.171875,
+          "logprob": -11.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2265625,
+          "logprob": -11.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8984375,
           "text": "<image>"
         },
         {
@@ -57919,137 +42220,37 @@
         },
         {
           "id": 32000,
-          "logprob": -11.7578125,
+          "logprob": -11.328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.875,
+          "logprob": -11.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4453125,
+          "logprob": -11.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1796875,
+          "logprob": -15.046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9453125,
+          "logprob": -12.2109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8203125,
+          "logprob": -12.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4765625,
+          "logprob": -16.875,
           "text": "<image>"
         },
         {
@@ -58059,22 +42260,227 @@
         },
         {
           "id": 32000,
-          "logprob": -15.484375,
+          "logprob": -14.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.6328125,
+          "logprob": -11.9609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.6015625,
+          "logprob": -12.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.75,
+          "logprob": -11.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.890625,
           "text": "<image>"
         },
         {
@@ -58084,7 +42490,807 @@
         },
         {
           "id": 32000,
-          "logprob": -13.3125,
+          "logprob": -13.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -18.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.0859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2578125,
           "text": "<image>"
         },
         {
@@ -58094,12 +43300,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.484375,
+          "logprob": -13.5390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2734375,
+          "logprob": -13.265625,
           "text": "<image>"
         },
         {
@@ -58114,17 +43320,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.6796875,
+          "logprob": -11.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.53125,
+          "logprob": -16.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1484375,
+          "logprob": -14.171875,
           "text": "<image>"
         },
         {
@@ -58134,17 +43340,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8125,
+          "logprob": -11.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.03125,
+          "logprob": -15.1171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.484375,
+          "logprob": -15.4453125,
           "text": "<image>"
         },
         {
@@ -58159,37 +43365,37 @@
         },
         {
           "id": 32000,
-          "logprob": -10.6953125,
+          "logprob": -10.703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1640625,
+          "logprob": -11.15625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.84375,
+          "logprob": -12.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5625,
+          "logprob": -12.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7421875,
+          "logprob": -12.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -18.0,
+          "logprob": -17.953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5859375,
+          "logprob": -11.59375,
           "text": "<image>"
         },
         {
@@ -58204,12 +43410,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.6796875,
+          "logprob": -11.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.71875,
+          "logprob": -12.8515625,
           "text": "<image>"
         },
         {
@@ -58219,27 +43425,27 @@
         },
         {
           "id": 32000,
-          "logprob": -12.1875,
+          "logprob": -12.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7578125,
+          "logprob": -12.7265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.125,
+          "logprob": -15.140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.9140625,
+          "logprob": -14.9375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.546875,
+          "logprob": -15.6875,
           "text": "<image>"
         },
         {
@@ -58249,22 +43455,22 @@
         },
         {
           "id": 32000,
-          "logprob": -14.109375,
+          "logprob": -14.125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.234375,
+          "logprob": -13.2265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.15625,
+          "logprob": -13.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.734375,
+          "logprob": -12.703125,
           "text": "<image>"
         },
         {
@@ -58274,7 +43480,7 @@
         },
         {
           "id": 32000,
-          "logprob": -14.7734375,
+          "logprob": -14.765625,
           "text": "<image>"
         },
         {
@@ -58284,12 +43490,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.7421875,
+          "logprob": -11.71875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.921875,
+          "logprob": -11.9296875,
           "text": "<image>"
         },
         {
@@ -58299,17 +43505,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.6015625,
+          "logprob": -13.6328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4375,
+          "logprob": -11.4296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5,
+          "logprob": -13.5078125,
           "text": "<image>"
         },
         {
@@ -58324,27 +43530,27 @@
         },
         {
           "id": 32000,
-          "logprob": -12.1328125,
+          "logprob": -12.15625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.90625,
+          "logprob": -13.1171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.578125,
+          "logprob": -12.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3359375,
+          "logprob": -13.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7265625,
+          "logprob": -11.734375,
           "text": "<image>"
         },
         {
@@ -58352,491 +43558,11 @@
           "logprob": -10.953125,
           "text": "<image>"
         },
-        {
-          "id": 32000,
-          "logprob": -12.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -19.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.796875,
-          "text": "<image>"
-        },
         {
           "id": 32000,
           "logprob": -12.75,
           "text": "<image>"
         },
-        {
-          "id": 32000,
-          "logprob": -14.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.1953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.515625,
-          "text": "<image>"
-        },
         {
           "id": 32000,
           "logprob": -12.7734375,
@@ -58844,7 +43570,22 @@
         },
         {
           "id": 32000,
-          "logprob": -12.4609375,
+          "logprob": -13.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5390625,
           "text": "<image>"
         },
         {
@@ -58854,87 +43595,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.171875,
+          "logprob": -19.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8828125,
+          "logprob": -12.4765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8828125,
+          "logprob": -11.609375,
           "text": "<image>"
         },
         {
@@ -58947,6 +43618,541 @@
           "logprob": -13.3203125,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -12.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.359375,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -13.2109375,
@@ -58954,7 +44160,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.9765625,
+          "logprob": -11.984375,
           "text": "<image>"
         },
         {
@@ -58964,12 +44170,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0078125,
+          "logprob": -12.015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0390625,
+          "logprob": -11.03125,
           "text": "<image>"
         },
         {
@@ -58984,17 +44190,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.8203125,
+          "logprob": -13.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5078125,
+          "logprob": -13.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.734375,
+          "logprob": -11.7265625,
           "text": "<image>"
         },
         {
@@ -59004,22 +44210,22 @@
         },
         {
           "id": 32000,
-          "logprob": -17.3125,
+          "logprob": -16.84375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5234375,
+          "logprob": -12.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -17.625,
+          "logprob": -17.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9296875,
+          "logprob": -11.9453125,
           "text": "<image>"
         },
         {
@@ -59029,17 +44235,17 @@
         },
         {
           "id": 32000,
-          "logprob": -15.9140625,
+          "logprob": -16.0,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.65625,
+          "logprob": -16.625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5,
+          "logprob": -12.4921875,
           "text": "<image>"
         },
         {
@@ -59049,17 +44255,17 @@
         },
         {
           "id": 368,
-          "logprob": -0.19726562,
+          "logprob": -0.19604492,
           "text": "you"
         },
         {
           "id": 1912,
-          "logprob": -1.4990234,
+          "logprob": -1.5058594,
           "text": "tell"
         },
         {
           "id": 528,
-          "logprob": -0.31152344,
+          "logprob": -0.31030273,
           "text": "me"
         },
         {
@@ -59074,22 +44280,22 @@
         },
         {
           "id": 2485,
-          "logprob": -0.9941406,
+          "logprob": -0.9975586,
           "text": "short"
         },
         {
           "id": 2838,
-          "logprob": -0.46118164,
+          "logprob": -0.4633789,
           "text": "story"
         },
         {
           "id": 2818,
-          "logprob": -3.3183594,
+          "logprob": -3.3144531,
           "text": "based"
         },
         {
           "id": 356,
-          "logprob": -0.029129028,
+          "logprob": -0.029037476,
           "text": "on"
         },
         {
@@ -59099,12 +44305,12 @@
         },
         {
           "id": 3469,
-          "logprob": -0.29052734,
+          "logprob": -0.2890625,
           "text": "image"
         },
         {
           "id": 28804,
-          "logprob": -0.43188477,
+          "logprob": -0.42895508,
           "text": "?"
         }
       ],
@@ -59112,13 +44318,14807 @@
       "tokens": [
         {
           "id": 13,
-          "logprob": -0.0076828003,
+          "logprob": -0.007621765,
           "special": false,
           "text": "\n"
         },
         {
           "id": 13,
-          "logprob": -0.19958496,
+          "logprob": -0.20275879,
+          "special": false,
+          "text": "\n"
+        },
+        {
+          "id": 16114,
+          "logprob": -1.2578125,
+          "special": false,
+          "text": "Once"
+        },
+        {
+          "id": 3714,
+          "logprob": -0.2084961,
+          "special": false,
+          "text": " upon"
+        },
+        {
+          "id": 264,
+          "logprob": -0.0017738342,
+          "special": false,
+          "text": " a"
+        },
+        {
+          "id": 727,
+          "logprob": -0.011932373,
+          "special": false,
+          "text": " time"
+        },
+        {
+          "id": 28725,
+          "logprob": -0.17297363,
+          "special": false,
+          "text": ","
+        },
+        {
+          "id": 736,
+          "logprob": -0.9057617,
+          "special": false,
+          "text": " there"
+        },
+        {
+          "id": 403,
+          "logprob": -0.05758667,
+          "special": false,
+          "text": " was"
+        },
+        {
+          "id": 264,
+          "logprob": -0.00970459,
+          "special": false,
+          "text": " a"
+        }
+      ],
+      "top_tokens": null
+    },
+    "generated_text": "\n\nOnce upon a time, there was a"
+  },
+  {
+    "details": {
+      "best_of_sequences": null,
+      "finish_reason": "length",
+      "generated_tokens": 10,
+      "prefill": [
+        {
+          "id": 1,
+          "logprob": null,
+          "text": "<s>"
+        },
+        {
+          "id": 1247,
+          "logprob": -2.390625,
+          "text": "User"
+        },
+        {
+          "id": 28747,
+          "logprob": -12.328125,
+          "text": ":"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.4921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.1015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.4609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -9.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -18.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.5234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.2265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -9.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.1484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.4609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.0859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.4609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.4609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.8984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -9.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -9.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.4296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.4921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -18.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.4921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.2265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.4296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.1484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.1484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.4296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.4921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -9.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.2265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.1484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.4296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.4921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -9.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.4609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -19.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -18.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.0859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -19.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4921875,
+          "text": "<image>"
+        },
+        {
+          "id": 2418,
+          "logprob": -19.0625,
+          "text": "Can"
+        },
+        {
+          "id": 368,
+          "logprob": -0.19604492,
+          "text": "you"
+        },
+        {
+          "id": 1912,
+          "logprob": -1.5058594,
+          "text": "tell"
+        },
+        {
+          "id": 528,
+          "logprob": -0.31030273,
+          "text": "me"
+        },
+        {
+          "id": 264,
+          "logprob": -2.6367188,
+          "text": "a"
+        },
+        {
+          "id": 1215,
+          "logprob": -9.1015625,
+          "text": "very"
+        },
+        {
+          "id": 2485,
+          "logprob": -0.9975586,
+          "text": "short"
+        },
+        {
+          "id": 2838,
+          "logprob": -0.4633789,
+          "text": "story"
+        },
+        {
+          "id": 2818,
+          "logprob": -3.3144531,
+          "text": "based"
+        },
+        {
+          "id": 356,
+          "logprob": -0.029037476,
+          "text": "on"
+        },
+        {
+          "id": 272,
+          "logprob": -0.9902344,
+          "text": "the"
+        },
+        {
+          "id": 3469,
+          "logprob": -0.2890625,
+          "text": "image"
+        },
+        {
+          "id": 28804,
+          "logprob": -0.42895508,
+          "text": "?"
+        }
+      ],
+      "seed": null,
+      "tokens": [
+        {
+          "id": 13,
+          "logprob": -0.007621765,
+          "special": false,
+          "text": "\n"
+        },
+        {
+          "id": 13,
+          "logprob": -0.20812988,
           "special": false,
           "text": "\n"
         },
@@ -59130,37 +59130,37 @@
         },
         {
           "id": 3714,
-          "logprob": -0.20861816,
+          "logprob": -0.20825195,
           "special": false,
           "text": " upon"
         },
         {
           "id": 264,
-          "logprob": -0.0017719269,
+          "logprob": -0.0017709732,
           "special": false,
           "text": " a"
         },
         {
           "id": 727,
-          "logprob": -0.011749268,
+          "logprob": -0.011932373,
           "special": false,
           "text": " time"
         },
         {
           "id": 28725,
-          "logprob": -0.17529297,
+          "logprob": -0.17297363,
           "special": false,
           "text": ","
         },
         {
           "id": 736,
-          "logprob": -0.9086914,
+          "logprob": -0.9057617,
           "special": false,
           "text": " there"
         },
         {
           "id": 403,
-          "logprob": -0.056732178,
+          "logprob": -0.05758667,
           "special": false,
           "text": " was"
         },
diff --git a/integration-tests/models/__snapshots__/test_lora_mistral/test_lora_mistral_with_customer_support_adapter.json b/integration-tests/models/__snapshots__/test_lora_mistral/test_lora_mistral_with_customer_support_adapter.json
new file mode 100644
index 00000000..dfdd2cc3
--- /dev/null
+++ b/integration-tests/models/__snapshots__/test_lora_mistral/test_lora_mistral_with_customer_support_adapter.json
@@ -0,0 +1,251 @@
+{
+  "details": {
+    "finish_reason": "length",
+    "generated_tokens": 40,
+    "prefill": [],
+    "seed": null,
+    "tokens": [
+      {
+        "id": 13,
+        "logprob": -0.27416992,
+        "special": false,
+        "text": "\n"
+      },
+      {
+        "id": 13,
+        "logprob": -0.17016602,
+        "special": false,
+        "text": "\n"
+      },
+      {
+        "id": 28737,
+        "logprob": -2.7109375,
+        "special": false,
+        "text": "I"
+      },
+      {
+        "id": 28809,
+        "logprob": -1.5,
+        "special": false,
+        "text": "’"
+      },
+      {
+        "id": 28719,
+        "logprob": -0.34204102,
+        "special": false,
+        "text": "m"
+      },
+      {
+        "id": 459,
+        "logprob": -1.6914062,
+        "special": false,
+        "text": " not"
+      },
+      {
+        "id": 1864,
+        "logprob": -0.69140625,
+        "special": false,
+        "text": " sure"
+      },
+      {
+        "id": 513,
+        "logprob": -1.6171875,
+        "special": false,
+        "text": " if"
+      },
+      {
+        "id": 315,
+        "logprob": -1.3837891,
+        "special": false,
+        "text": " I"
+      },
+      {
+        "id": 541,
+        "logprob": -1.2226562,
+        "special": false,
+        "text": " can"
+      },
+      {
+        "id": 1567,
+        "logprob": -1.8652344,
+        "special": false,
+        "text": " come"
+      },
+      {
+        "id": 582,
+        "logprob": -0.0070228577,
+        "special": false,
+        "text": " up"
+      },
+      {
+        "id": 395,
+        "logprob": -0.0054092407,
+        "special": false,
+        "text": " with"
+      },
+      {
+        "id": 28705,
+        "logprob": -0.62597656,
+        "special": false,
+        "text": " "
+      },
+      {
+        "id": 28770,
+        "logprob": -0.0035572052,
+        "special": false,
+        "text": "3"
+      },
+      {
+        "id": 4842,
+        "logprob": -0.93603516,
+        "special": false,
+        "text": " unique"
+      },
+      {
+        "id": 3085,
+        "logprob": -0.028411865,
+        "special": false,
+        "text": " words"
+      },
+      {
+        "id": 369,
+        "logprob": -1.0400391,
+        "special": false,
+        "text": " that"
+      },
+      {
+        "id": 6685,
+        "logprob": -0.09710693,
+        "special": false,
+        "text": " describe"
+      },
+      {
+        "id": 528,
+        "logprob": -0.066467285,
+        "special": false,
+        "text": " me"
+      },
+      {
+        "id": 28725,
+        "logprob": -1.0722656,
+        "special": false,
+        "text": ","
+      },
+      {
+        "id": 562,
+        "logprob": -0.33422852,
+        "special": false,
+        "text": " but"
+      },
+      {
+        "id": 315,
+        "logprob": -0.5136719,
+        "special": false,
+        "text": " I"
+      },
+      {
+        "id": 28809,
+        "logprob": -0.8989258,
+        "special": false,
+        "text": "’"
+      },
+      {
+        "id": 584,
+        "logprob": -0.2076416,
+        "special": false,
+        "text": "ll"
+      },
+      {
+        "id": 1464,
+        "logprob": -0.8808594,
+        "special": false,
+        "text": " try"
+      },
+      {
+        "id": 28723,
+        "logprob": -0.88427734,
+        "special": false,
+        "text": "."
+      },
+      {
+        "id": 13,
+        "logprob": -0.91064453,
+        "special": false,
+        "text": "\n"
+      },
+      {
+        "id": 13,
+        "logprob": -0.08105469,
+        "special": false,
+        "text": "\n"
+      },
+      {
+        "id": 28740,
+        "logprob": -1.8486328,
+        "special": false,
+        "text": "1"
+      },
+      {
+        "id": 28723,
+        "logprob": -0.111572266,
+        "special": false,
+        "text": "."
+      },
+      {
+        "id": 23626,
+        "logprob": -3.15625,
+        "special": false,
+        "text": " Creative"
+      },
+      {
+        "id": 13,
+        "logprob": -0.9194336,
+        "special": false,
+        "text": "\n"
+      },
+      {
+        "id": 28750,
+        "logprob": -0.24841309,
+        "special": false,
+        "text": "2"
+      },
+      {
+        "id": 28723,
+        "logprob": -9.393692e-05,
+        "special": false,
+        "text": "."
+      },
+      {
+        "id": 6785,
+        "logprob": -3.1386719,
+        "special": false,
+        "text": " Fun"
+      },
+      {
+        "id": 1780,
+        "logprob": -0.53564453,
+        "special": false,
+        "text": "ny"
+      },
+      {
+        "id": 13,
+        "logprob": -0.09033203,
+        "special": false,
+        "text": "\n"
+      },
+      {
+        "id": 28770,
+        "logprob": -0.00466156,
+        "special": false,
+        "text": "3"
+      },
+      {
+        "id": 28723,
+        "logprob": -0.00016450882,
+        "special": false,
+        "text": "."
+      }
+    ]
+  },
+  "generated_text": "\n\nI’m not sure if I can come up with 3 unique words that describe me, but I’ll try.\n\n1. Creative\n2. Funny\n3."
+}
diff --git a/integration-tests/models/__snapshots__/test_lora_mistral/test_lora_mistral_with_dbpedia_adapter.json b/integration-tests/models/__snapshots__/test_lora_mistral/test_lora_mistral_with_dbpedia_adapter.json
new file mode 100644
index 00000000..91eb5edf
--- /dev/null
+++ b/integration-tests/models/__snapshots__/test_lora_mistral/test_lora_mistral_with_dbpedia_adapter.json
@@ -0,0 +1,53 @@
+{
+  "details": {
+    "finish_reason": "eos_token",
+    "generated_tokens": 7,
+    "prefill": [],
+    "seed": null,
+    "tokens": [
+      {
+        "id": 1,
+        "logprob": -0.49658203,
+        "special": true,
+        "text": "<s>"
+      },
+      {
+        "id": 28705,
+        "logprob": -0.0016384125,
+        "special": false,
+        "text": " "
+      },
+      {
+        "id": 1,
+        "logprob": -1.4931641,
+        "special": true,
+        "text": "<s>"
+      },
+      {
+        "id": 28705,
+        "logprob": -0.00075769424,
+        "special": false,
+        "text": " "
+      },
+      {
+        "id": 28740,
+        "logprob": -0.25024414,
+        "special": false,
+        "text": "1"
+      },
+      {
+        "id": 28740,
+        "logprob": -0.2631836,
+        "special": false,
+        "text": "1"
+      },
+      {
+        "id": 2,
+        "logprob": -0.0003285408,
+        "special": true,
+        "text": "</s>"
+      }
+    ]
+  },
+  "generated_text": "  11"
+}
diff --git a/integration-tests/models/__snapshots__/test_lora_mistral/test_lora_mistral_without_adapter.json b/integration-tests/models/__snapshots__/test_lora_mistral/test_lora_mistral_without_adapter.json
new file mode 100644
index 00000000..13018688
--- /dev/null
+++ b/integration-tests/models/__snapshots__/test_lora_mistral/test_lora_mistral_without_adapter.json
@@ -0,0 +1,251 @@
+{
+  "details": {
+    "finish_reason": "length",
+    "generated_tokens": 40,
+    "prefill": [],
+    "seed": null,
+    "tokens": [
+      {
+        "id": 13,
+        "logprob": -1.0488281,
+        "special": false,
+        "text": "\n"
+      },
+      {
+        "id": 13,
+        "logprob": -1.0800781,
+        "special": false,
+        "text": "\n"
+      },
+      {
+        "id": 27332,
+        "logprob": -2.1152344,
+        "special": false,
+        "text": "###"
+      },
+      {
+        "id": 28705,
+        "logprob": -1.6748047,
+        "special": false,
+        "text": " "
+      },
+      {
+        "id": 28740,
+        "logprob": -0.097229004,
+        "special": false,
+        "text": "1"
+      },
+      {
+        "id": 28723,
+        "logprob": -0.16467285,
+        "special": false,
+        "text": "."
+      },
+      {
+        "id": 7615,
+        "logprob": -2.2246094,
+        "special": false,
+        "text": " News"
+      },
+      {
+        "id": 13,
+        "logprob": -1.0488281,
+        "special": false,
+        "text": "\n"
+      },
+      {
+        "id": 27332,
+        "logprob": -0.69189453,
+        "special": false,
+        "text": "###"
+      },
+      {
+        "id": 28705,
+        "logprob": -0.013343811,
+        "special": false,
+        "text": " "
+      },
+      {
+        "id": 28750,
+        "logprob": -0.011230469,
+        "special": false,
+        "text": "2"
+      },
+      {
+        "id": 28723,
+        "logprob": -0.00096845627,
+        "special": false,
+        "text": "."
+      },
+      {
+        "id": 21095,
+        "logprob": -2.5605469,
+        "special": false,
+        "text": " Blog"
+      },
+      {
+        "id": 13,
+        "logprob": -0.19458008,
+        "special": false,
+        "text": "\n"
+      },
+      {
+        "id": 27332,
+        "logprob": -0.031280518,
+        "special": false,
+        "text": "###"
+      },
+      {
+        "id": 28705,
+        "logprob": -0.0030708313,
+        "special": false,
+        "text": " "
+      },
+      {
+        "id": 28770,
+        "logprob": -0.0029277802,
+        "special": false,
+        "text": "3"
+      },
+      {
+        "id": 28723,
+        "logprob": -0.0012350082,
+        "special": false,
+        "text": "."
+      },
+      {
+        "id": 20108,
+        "logprob": -2.1582031,
+        "special": false,
+        "text": " Article"
+      },
+      {
+        "id": 13,
+        "logprob": -0.05810547,
+        "special": false,
+        "text": "\n"
+      },
+      {
+        "id": 27332,
+        "logprob": -0.35083008,
+        "special": false,
+        "text": "###"
+      },
+      {
+        "id": 28705,
+        "logprob": -0.034332275,
+        "special": false,
+        "text": " "
+      },
+      {
+        "id": 28781,
+        "logprob": -0.009666443,
+        "special": false,
+        "text": "4"
+      },
+      {
+        "id": 28723,
+        "logprob": -0.0013113022,
+        "special": false,
+        "text": "."
+      },
+      {
+        "id": 8349,
+        "logprob": -2.6191406,
+        "special": false,
+        "text": " Review"
+      },
+      {
+        "id": 13,
+        "logprob": -0.04031372,
+        "special": false,
+        "text": "\n"
+      },
+      {
+        "id": 27332,
+        "logprob": -0.45239258,
+        "special": false,
+        "text": "###"
+      },
+      {
+        "id": 28705,
+        "logprob": -0.045410156,
+        "special": false,
+        "text": " "
+      },
+      {
+        "id": 28782,
+        "logprob": -0.0041236877,
+        "special": false,
+        "text": "5"
+      },
+      {
+        "id": 28723,
+        "logprob": -0.0010223389,
+        "special": false,
+        "text": "."
+      },
+      {
+        "id": 5299,
+        "logprob": -2.8066406,
+        "special": false,
+        "text": " Other"
+      },
+      {
+        "id": 13,
+        "logprob": -0.12054443,
+        "special": false,
+        "text": "\n"
+      },
+      {
+        "id": 13,
+        "logprob": -0.44580078,
+        "special": false,
+        "text": "\n"
+      },
+      {
+        "id": 13,
+        "logprob": -1.4921875,
+        "special": false,
+        "text": "\n"
+      },
+      {
+        "id": 13,
+        "logprob": -1.3574219,
+        "special": false,
+        "text": "\n"
+      },
+      {
+        "id": 13,
+        "logprob": -1.0039062,
+        "special": false,
+        "text": "\n"
+      },
+      {
+        "id": 13,
+        "logprob": -0.5859375,
+        "special": false,
+        "text": "\n"
+      },
+      {
+        "id": 13,
+        "logprob": -0.43481445,
+        "special": false,
+        "text": "\n"
+      },
+      {
+        "id": 13,
+        "logprob": -0.2783203,
+        "special": false,
+        "text": "\n"
+      },
+      {
+        "id": 13,
+        "logprob": -0.20410156,
+        "special": false,
+        "text": "\n"
+      }
+    ]
+  },
+  "generated_text": "\n\n### 1. News\n### 2. Blog\n### 3. Article\n### 4. Review\n### 5. Other\n\n\n\n\n\n\n\n\n"
+}
diff --git a/integration-tests/models/__snapshots__/test_lora_mistral/test_lora_mistral_without_customer_support_adapter.json b/integration-tests/models/__snapshots__/test_lora_mistral/test_lora_mistral_without_customer_support_adapter.json
new file mode 100644
index 00000000..8c00dee7
--- /dev/null
+++ b/integration-tests/models/__snapshots__/test_lora_mistral/test_lora_mistral_without_customer_support_adapter.json
@@ -0,0 +1,251 @@
+{
+  "details": {
+    "finish_reason": "length",
+    "generated_tokens": 40,
+    "prefill": [],
+    "seed": null,
+    "tokens": [
+      {
+        "id": 13,
+        "logprob": -0.31347656,
+        "special": false,
+        "text": "\n"
+      },
+      {
+        "id": 13,
+        "logprob": -0.27441406,
+        "special": false,
+        "text": "\n"
+      },
+      {
+        "id": 28737,
+        "logprob": -2.2285156,
+        "special": false,
+        "text": "I"
+      },
+      {
+        "id": 28809,
+        "logprob": -1.4677734,
+        "special": false,
+        "text": "’"
+      },
+      {
+        "id": 28719,
+        "logprob": -0.31762695,
+        "special": false,
+        "text": "m"
+      },
+      {
+        "id": 264,
+        "logprob": -1.6865234,
+        "special": false,
+        "text": " a"
+      },
+      {
+        "id": 1215,
+        "logprob": -3.2695312,
+        "special": false,
+        "text": " very"
+      },
+      {
+        "id": 20640,
+        "logprob": -3.1230469,
+        "special": false,
+        "text": " passionate"
+      },
+      {
+        "id": 1338,
+        "logprob": -0.48339844,
+        "special": false,
+        "text": " person"
+      },
+      {
+        "id": 28723,
+        "logprob": -0.9970703,
+        "special": false,
+        "text": "."
+      },
+      {
+        "id": 315,
+        "logprob": -0.5498047,
+        "special": false,
+        "text": " I"
+      },
+      {
+        "id": 28809,
+        "logprob": -1.1923828,
+        "special": false,
+        "text": "’"
+      },
+      {
+        "id": 28719,
+        "logprob": -0.080444336,
+        "special": false,
+        "text": "m"
+      },
+      {
+        "id": 1215,
+        "logprob": -1.8271484,
+        "special": false,
+        "text": " very"
+      },
+      {
+        "id": 12215,
+        "logprob": -2.8847656,
+        "special": false,
+        "text": " driven"
+      },
+      {
+        "id": 28723,
+        "logprob": -1.0927734,
+        "special": false,
+        "text": "."
+      },
+      {
+        "id": 315,
+        "logprob": -0.4584961,
+        "special": false,
+        "text": " I"
+      },
+      {
+        "id": 28809,
+        "logprob": -0.5019531,
+        "special": false,
+        "text": "’"
+      },
+      {
+        "id": 28719,
+        "logprob": -0.030715942,
+        "special": false,
+        "text": "m"
+      },
+      {
+        "id": 1215,
+        "logprob": -0.96972656,
+        "special": false,
+        "text": " very"
+      },
+      {
+        "id": 7798,
+        "logprob": -2.8847656,
+        "special": false,
+        "text": " determined"
+      },
+      {
+        "id": 28723,
+        "logprob": -0.27319336,
+        "special": false,
+        "text": "."
+      },
+      {
+        "id": 13,
+        "logprob": -0.56396484,
+        "special": false,
+        "text": "\n"
+      },
+      {
+        "id": 13,
+        "logprob": -0.011016846,
+        "special": false,
+        "text": "\n"
+      },
+      {
+        "id": 3195,
+        "logprob": -0.7163086,
+        "special": false,
+        "text": "What"
+      },
+      {
+        "id": 349,
+        "logprob": -1.1611328,
+        "special": false,
+        "text": " is"
+      },
+      {
+        "id": 574,
+        "logprob": -0.515625,
+        "special": false,
+        "text": " your"
+      },
+      {
+        "id": 6656,
+        "logprob": -1.0253906,
+        "special": false,
+        "text": " favorite"
+      },
+      {
+        "id": 1970,
+        "logprob": -2.1738281,
+        "special": false,
+        "text": " thing"
+      },
+      {
+        "id": 684,
+        "logprob": -0.48364258,
+        "special": false,
+        "text": " about"
+      },
+      {
+        "id": 1250,
+        "logprob": -1.8876953,
+        "special": false,
+        "text": " being"
+      },
+      {
+        "id": 264,
+        "logprob": -0.41967773,
+        "special": false,
+        "text": " a"
+      },
+      {
+        "id": 8626,
+        "logprob": -2.9160156,
+        "special": false,
+        "text": " teacher"
+      },
+      {
+        "id": 28804,
+        "logprob": -0.11920166,
+        "special": false,
+        "text": "?"
+      },
+      {
+        "id": 13,
+        "logprob": -0.023727417,
+        "special": false,
+        "text": "\n"
+      },
+      {
+        "id": 13,
+        "logprob": -0.010848999,
+        "special": false,
+        "text": "\n"
+      },
+      {
+        "id": 28737,
+        "logprob": -1.0566406,
+        "special": false,
+        "text": "I"
+      },
+      {
+        "id": 2016,
+        "logprob": -0.7163086,
+        "special": false,
+        "text": " love"
+      },
+      {
+        "id": 272,
+        "logprob": -1.9169922,
+        "special": false,
+        "text": " the"
+      },
+      {
+        "id": 1639,
+        "logprob": -2.03125,
+        "special": false,
+        "text": " fact"
+      }
+    ]
+  },
+  "generated_text": "\n\nI’m a very passionate person. I’m very driven. I’m very determined.\n\nWhat is your favorite thing about being a teacher?\n\nI love the fact"
+}
diff --git a/integration-tests/models/__snapshots__/test_mamba/test_mamba.json b/integration-tests/models/__snapshots__/test_mamba/test_mamba.json
index eaba5078..9079b3bd 100644
--- a/integration-tests/models/__snapshots__/test_mamba/test_mamba.json
+++ b/integration-tests/models/__snapshots__/test_mamba/test_mamba.json
@@ -14,55 +14,55 @@
       },
       {
         "id": 187,
-        "logprob": -0.26953125,
+        "logprob": -0.35742188,
         "special": false,
         "text": "\n"
       },
       {
         "id": 30763,
-        "logprob": -1.1953125,
+        "logprob": -1.1015625,
         "special": false,
         "text": "Deep"
       },
       {
         "id": 4715,
-        "logprob": -0.53515625,
+        "logprob": -0.5234375,
         "special": false,
         "text": " learning"
       },
       {
         "id": 310,
-        "logprob": -0.625,
+        "logprob": -0.55078125,
         "special": false,
         "text": " is"
       },
       {
         "id": 247,
-        "logprob": -0.6796875,
+        "logprob": -0.6640625,
         "special": false,
         "text": " a"
       },
       {
         "id": 747,
-        "logprob": -2.0,
+        "logprob": -2.0625,
         "special": false,
         "text": " new"
       },
       {
         "id": 1511,
-        "logprob": -2.3125,
+        "logprob": -2.375,
         "special": false,
         "text": " type"
       },
       {
         "id": 273,
-        "logprob": -0.0028533936,
+        "logprob": -0.0029144287,
         "special": false,
         "text": " of"
       },
       {
         "id": 5145,
-        "logprob": -1.265625,
+        "logprob": -1.2734375,
         "special": false,
         "text": " machine"
       }
diff --git a/integration-tests/models/__snapshots__/test_mamba/test_mamba_all_params.json b/integration-tests/models/__snapshots__/test_mamba/test_mamba_all_params.json
index 85e9a9e0..ef88926c 100644
--- a/integration-tests/models/__snapshots__/test_mamba/test_mamba_all_params.json
+++ b/integration-tests/models/__snapshots__/test_mamba/test_mamba_all_params.json
@@ -52,7 +52,7 @@
       },
       {
         "id": 9830,
-        "logprob": -1.65625,
+        "logprob": -2.25,
         "special": false,
         "text": " colors"
       },
@@ -64,13 +64,13 @@
       },
       {
         "id": 329,
-        "logprob": -2.4375,
+        "logprob": -2.171875,
         "special": false,
         "text": " A"
       },
       {
         "id": 1180,
-        "logprob": -1.953125,
+        "logprob": -2.046875,
         "special": false,
         "text": " number"
       },
diff --git a/integration-tests/models/__snapshots__/test_mllama/test_mllama_load.json b/integration-tests/models/__snapshots__/test_mllama/test_mllama_load.json
new file mode 100644
index 00000000..95ba7a78
--- /dev/null
+++ b/integration-tests/models/__snapshots__/test_mllama/test_mllama_load.json
@@ -0,0 +1,106 @@
+[
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "In a bustling city, a chicken named Cluck",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1727773835,
+    "id": "",
+    "model": "meta-llama/Llama-3.2-11B-Vision-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.3.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 50,
+      "total_tokens": 60
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "In a world where even chickens could dream big,",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1727773835,
+    "id": "",
+    "model": "meta-llama/Llama-3.2-11B-Vision-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.3.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 50,
+      "total_tokens": 60
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "In a world where even chickens could dream big,",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1727773835,
+    "id": "",
+    "model": "meta-llama/Llama-3.2-11B-Vision-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.3.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 50,
+      "total_tokens": 60
+    }
+  },
+  {
+    "choices": [
+      {
+        "finish_reason": "length",
+        "index": 0,
+        "logprobs": null,
+        "message": {
+          "content": "In a world where even chickens could dream big,",
+          "name": null,
+          "role": "assistant",
+          "tool_calls": null
+        },
+        "usage": null
+      }
+    ],
+    "created": 1727773835,
+    "id": "",
+    "model": "meta-llama/Llama-3.2-11B-Vision-Instruct",
+    "object": "chat.completion",
+    "system_fingerprint": "2.3.1-dev0-native",
+    "usage": {
+      "completion_tokens": 10,
+      "prompt_tokens": 50,
+      "total_tokens": 60
+    }
+  }
+]
diff --git a/integration-tests/models/__snapshots__/test_mllama/test_mllama_simpl.json b/integration-tests/models/__snapshots__/test_mllama/test_mllama_simpl.json
new file mode 100644
index 00000000..1d4dd6d7
--- /dev/null
+++ b/integration-tests/models/__snapshots__/test_mllama/test_mllama_simpl.json
@@ -0,0 +1,26 @@
+{
+  "choices": [
+    {
+      "finish_reason": "length",
+      "index": 0,
+      "logprobs": null,
+      "message": {
+        "content": "In a bustling city, a chicken named Cluck",
+        "name": null,
+        "role": "assistant",
+        "tool_calls": null
+      },
+      "usage": null
+    }
+  ],
+  "created": 1727556016,
+  "id": "",
+  "model": "meta-llama/Llama-3.2-11B-Vision-Instruct",
+  "object": "chat.completion",
+  "system_fingerprint": "2.3.1-dev0-native",
+  "usage": {
+    "completion_tokens": 10,
+    "prompt_tokens": 50,
+    "total_tokens": 60
+  }
+}
diff --git a/integration-tests/models/__snapshots__/test_mt0_base/test_mt0_base_all_params.json b/integration-tests/models/__snapshots__/test_mt0_base/test_mt0_base_all_params.json
index 5cacf3e9..40ec7e2f 100644
--- a/integration-tests/models/__snapshots__/test_mt0_base/test_mt0_base_all_params.json
+++ b/integration-tests/models/__snapshots__/test_mt0_base/test_mt0_base_all_params.json
@@ -26,13 +26,13 @@
       },
       {
         "id": 259,
-        "logprob": -0.4716797,
+        "logprob": -0.46948242,
         "special": false,
         "text": " "
       },
       {
         "id": 261,
-        "logprob": -0.044677734,
+        "logprob": -0.15307617,
         "special": false,
         "text": ","
       },
@@ -56,7 +56,7 @@
       },
       {
         "id": 35622,
-        "logprob": -1.1630859,
+        "logprob": -1.2998047,
         "special": false,
         "text": " cloud"
       },
diff --git a/integration-tests/models/__snapshots__/test_server_gptq_quantized/test_server_gptq_quantized.json b/integration-tests/models/__snapshots__/test_server_gptq_quantized/test_server_gptq_quantized.json
new file mode 100644
index 00000000..69c1f47d
--- /dev/null
+++ b/integration-tests/models/__snapshots__/test_server_gptq_quantized/test_server_gptq_quantized.json
@@ -0,0 +1,89 @@
+{
+  "details": {
+    "best_of_sequences": null,
+    "finish_reason": "length",
+    "generated_tokens": 10,
+    "prefill": [
+      {
+        "id": 1,
+        "logprob": null,
+        "text": "<s>"
+      },
+      {
+        "id": 4321,
+        "logprob": -9.8359375,
+        "text": "Test"
+      },
+      {
+        "id": 2009,
+        "logprob": -9.6171875,
+        "text": "request"
+      }
+    ],
+    "seed": null,
+    "tokens": [
+      {
+        "id": 13,
+        "logprob": -2.3417969,
+        "special": false,
+        "text": "\n"
+      },
+      {
+        "id": 3057,
+        "logprob": -1.8730469,
+        "special": false,
+        "text": "Test"
+      },
+      {
+        "id": 2009,
+        "logprob": -1.2626953,
+        "special": false,
+        "text": " request"
+      },
+      {
+        "id": 13,
+        "logprob": -1.7060547,
+        "special": false,
+        "text": "\n"
+      },
+      {
+        "id": 3057,
+        "logprob": -1.4482422,
+        "special": false,
+        "text": "Test"
+      },
+      {
+        "id": 2009,
+        "logprob": -0.15246582,
+        "special": false,
+        "text": " request"
+      },
+      {
+        "id": 13,
+        "logprob": -0.796875,
+        "special": false,
+        "text": "\n"
+      },
+      {
+        "id": 3057,
+        "logprob": -0.22766113,
+        "special": false,
+        "text": "Test"
+      },
+      {
+        "id": 2009,
+        "logprob": -0.007045746,
+        "special": false,
+        "text": " request"
+      },
+      {
+        "id": 13,
+        "logprob": -0.021759033,
+        "special": false,
+        "text": "\n"
+      }
+    ],
+    "top_tokens": null
+  },
+  "generated_text": "\nTest request\nTest request\nTest request\n"
+}
diff --git a/integration-tests/models/__snapshots__/test_server_gptq_quantized/test_server_gptq_quantized_all_params.json b/integration-tests/models/__snapshots__/test_server_gptq_quantized/test_server_gptq_quantized_all_params.json
new file mode 100644
index 00000000..9b5ee9ee
--- /dev/null
+++ b/integration-tests/models/__snapshots__/test_server_gptq_quantized/test_server_gptq_quantized_all_params.json
@@ -0,0 +1,89 @@
+{
+  "details": {
+    "best_of_sequences": null,
+    "finish_reason": "length",
+    "generated_tokens": 10,
+    "prefill": [
+      {
+        "id": 1,
+        "logprob": null,
+        "text": "<s>"
+      },
+      {
+        "id": 4321,
+        "logprob": -9.7890625,
+        "text": "Test"
+      },
+      {
+        "id": 2009,
+        "logprob": -9.625,
+        "text": "request"
+      }
+    ],
+    "seed": 0,
+    "tokens": [
+      {
+        "id": 29899,
+        "logprob": -1.4980469,
+        "special": false,
+        "text": "-"
+      },
+      {
+        "id": 1454,
+        "logprob": -0.19433594,
+        "special": false,
+        "text": "for"
+      },
+      {
+        "id": 29899,
+        "logprob": 0.0,
+        "special": false,
+        "text": "-"
+      },
+      {
+        "id": 9342,
+        "logprob": 0.0,
+        "special": false,
+        "text": "comment"
+      },
+      {
+        "id": 29901,
+        "logprob": 0.0,
+        "special": false,
+        "text": ":"
+      },
+      {
+        "id": 396,
+        "logprob": -0.27392578,
+        "special": false,
+        "text": " #"
+      },
+      {
+        "id": 29906,
+        "logprob": -0.49389648,
+        "special": false,
+        "text": "2"
+      },
+      {
+        "id": 29900,
+        "logprob": -0.81103516,
+        "special": false,
+        "text": "0"
+      },
+      {
+        "id": 29896,
+        "logprob": 0.0,
+        "special": false,
+        "text": "1"
+      },
+      {
+        "id": 29955,
+        "logprob": -1.0800781,
+        "special": false,
+        "text": "7"
+      }
+    ],
+    "top_tokens": null
+  },
+  "generated_text": "Test request-for-comment: #2017"
+}
diff --git a/integration-tests/models/__snapshots__/test_server_gptq_quantized/test_server_gptq_quantized_load.json b/integration-tests/models/__snapshots__/test_server_gptq_quantized/test_server_gptq_quantized_load.json
new file mode 100644
index 00000000..df975635
--- /dev/null
+++ b/integration-tests/models/__snapshots__/test_server_gptq_quantized/test_server_gptq_quantized_load.json
@@ -0,0 +1,358 @@
+[
+  {
+    "details": {
+      "best_of_sequences": null,
+      "finish_reason": "length",
+      "generated_tokens": 10,
+      "prefill": [
+        {
+          "id": 1,
+          "logprob": null,
+          "text": "<s>"
+        },
+        {
+          "id": 4321,
+          "logprob": -9.8828125,
+          "text": "Test"
+        },
+        {
+          "id": 2009,
+          "logprob": -9.5859375,
+          "text": "request"
+        }
+      ],
+      "seed": null,
+      "tokens": [
+        {
+          "id": 13,
+          "logprob": -2.3359375,
+          "special": false,
+          "text": "\n"
+        },
+        {
+          "id": 3057,
+          "logprob": -1.8623047,
+          "special": false,
+          "text": "Test"
+        },
+        {
+          "id": 2009,
+          "logprob": -1.2451172,
+          "special": false,
+          "text": " request"
+        },
+        {
+          "id": 13,
+          "logprob": -1.6923828,
+          "special": false,
+          "text": "\n"
+        },
+        {
+          "id": 3057,
+          "logprob": -1.4492188,
+          "special": false,
+          "text": "Test"
+        },
+        {
+          "id": 2009,
+          "logprob": -0.15197754,
+          "special": false,
+          "text": " request"
+        },
+        {
+          "id": 13,
+          "logprob": -0.8022461,
+          "special": false,
+          "text": "\n"
+        },
+        {
+          "id": 3057,
+          "logprob": -0.22583008,
+          "special": false,
+          "text": "Test"
+        },
+        {
+          "id": 2009,
+          "logprob": -0.007095337,
+          "special": false,
+          "text": " request"
+        },
+        {
+          "id": 13,
+          "logprob": -0.021652222,
+          "special": false,
+          "text": "\n"
+        }
+      ],
+      "top_tokens": null
+    },
+    "generated_text": "\nTest request\nTest request\nTest request\n"
+  },
+  {
+    "details": {
+      "best_of_sequences": null,
+      "finish_reason": "length",
+      "generated_tokens": 10,
+      "prefill": [
+        {
+          "id": 1,
+          "logprob": null,
+          "text": "<s>"
+        },
+        {
+          "id": 4321,
+          "logprob": -9.796875,
+          "text": "Test"
+        },
+        {
+          "id": 2009,
+          "logprob": -9.625,
+          "text": "request"
+        }
+      ],
+      "seed": null,
+      "tokens": [
+        {
+          "id": 13,
+          "logprob": -2.3476562,
+          "special": false,
+          "text": "\n"
+        },
+        {
+          "id": 3057,
+          "logprob": -1.8789062,
+          "special": false,
+          "text": "Test"
+        },
+        {
+          "id": 2009,
+          "logprob": -1.2734375,
+          "special": false,
+          "text": " request"
+        },
+        {
+          "id": 13,
+          "logprob": -1.703125,
+          "special": false,
+          "text": "\n"
+        },
+        {
+          "id": 3057,
+          "logprob": -1.4677734,
+          "special": false,
+          "text": "Test"
+        },
+        {
+          "id": 2009,
+          "logprob": -0.15454102,
+          "special": false,
+          "text": " request"
+        },
+        {
+          "id": 13,
+          "logprob": -0.7973633,
+          "special": false,
+          "text": "\n"
+        },
+        {
+          "id": 3057,
+          "logprob": -0.23278809,
+          "special": false,
+          "text": "Test"
+        },
+        {
+          "id": 2009,
+          "logprob": -0.006980896,
+          "special": false,
+          "text": " request"
+        },
+        {
+          "id": 13,
+          "logprob": -0.022033691,
+          "special": false,
+          "text": "\n"
+        }
+      ],
+      "top_tokens": null
+    },
+    "generated_text": "\nTest request\nTest request\nTest request\n"
+  },
+  {
+    "details": {
+      "best_of_sequences": null,
+      "finish_reason": "length",
+      "generated_tokens": 10,
+      "prefill": [
+        {
+          "id": 1,
+          "logprob": null,
+          "text": "<s>"
+        },
+        {
+          "id": 4321,
+          "logprob": -9.9296875,
+          "text": "Test"
+        },
+        {
+          "id": 2009,
+          "logprob": -9.5703125,
+          "text": "request"
+        }
+      ],
+      "seed": null,
+      "tokens": [
+        {
+          "id": 13,
+          "logprob": -2.3203125,
+          "special": false,
+          "text": "\n"
+        },
+        {
+          "id": 3057,
+          "logprob": -1.8486328,
+          "special": false,
+          "text": "Test"
+        },
+        {
+          "id": 2009,
+          "logprob": -1.2480469,
+          "special": false,
+          "text": " request"
+        },
+        {
+          "id": 13,
+          "logprob": -1.7060547,
+          "special": false,
+          "text": "\n"
+        },
+        {
+          "id": 3057,
+          "logprob": -1.4511719,
+          "special": false,
+          "text": "Test"
+        },
+        {
+          "id": 2009,
+          "logprob": -0.1529541,
+          "special": false,
+          "text": " request"
+        },
+        {
+          "id": 13,
+          "logprob": -0.81396484,
+          "special": false,
+          "text": "\n"
+        },
+        {
+          "id": 3057,
+          "logprob": -0.22180176,
+          "special": false,
+          "text": "Test"
+        },
+        {
+          "id": 2009,
+          "logprob": -0.007133484,
+          "special": false,
+          "text": " request"
+        },
+        {
+          "id": 13,
+          "logprob": -0.021835327,
+          "special": false,
+          "text": "\n"
+        }
+      ],
+      "top_tokens": null
+    },
+    "generated_text": "\nTest request\nTest request\nTest request\n"
+  },
+  {
+    "details": {
+      "best_of_sequences": null,
+      "finish_reason": "length",
+      "generated_tokens": 10,
+      "prefill": [
+        {
+          "id": 1,
+          "logprob": null,
+          "text": "<s>"
+        },
+        {
+          "id": 4321,
+          "logprob": -9.84375,
+          "text": "Test"
+        },
+        {
+          "id": 2009,
+          "logprob": -9.6171875,
+          "text": "request"
+        }
+      ],
+      "seed": null,
+      "tokens": [
+        {
+          "id": 13,
+          "logprob": -2.3261719,
+          "special": false,
+          "text": "\n"
+        },
+        {
+          "id": 3057,
+          "logprob": -1.8691406,
+          "special": false,
+          "text": "Test"
+        },
+        {
+          "id": 2009,
+          "logprob": -1.2597656,
+          "special": false,
+          "text": " request"
+        },
+        {
+          "id": 13,
+          "logprob": -1.7070312,
+          "special": false,
+          "text": "\n"
+        },
+        {
+          "id": 3057,
+          "logprob": -1.4550781,
+          "special": false,
+          "text": "Test"
+        },
+        {
+          "id": 2009,
+          "logprob": -0.1538086,
+          "special": false,
+          "text": " request"
+        },
+        {
+          "id": 13,
+          "logprob": -0.79345703,
+          "special": false,
+          "text": "\n"
+        },
+        {
+          "id": 3057,
+          "logprob": -0.22924805,
+          "special": false,
+          "text": "Test"
+        },
+        {
+          "id": 2009,
+          "logprob": -0.0070266724,
+          "special": false,
+          "text": " request"
+        },
+        {
+          "id": 13,
+          "logprob": -0.021942139,
+          "special": false,
+          "text": "\n"
+        }
+      ],
+      "top_tokens": null
+    },
+    "generated_text": "\nTest request\nTest request\nTest request\n"
+  }
+]
diff --git a/integration-tests/models/__snapshots__/test_tools_llama/test_flash_llama_grammar_tools_insufficient_information.json b/integration-tests/models/__snapshots__/test_tools_llama/test_flash_llama_grammar_tools_insufficient_information.json
index 0cd3c67f..7f7f7884 100644
--- a/integration-tests/models/__snapshots__/test_tools_llama/test_flash_llama_grammar_tools_insufficient_information.json
+++ b/integration-tests/models/__snapshots__/test_tools_llama/test_flash_llama_grammar_tools_insufficient_information.json
@@ -1,38 +1,26 @@
 {
   "choices": [
     {
-      "finish_reason": "eos_token",
+      "finish_reason": "stop",
       "index": 0,
       "logprobs": null,
       "message": {
-        "content": null,
+        "content": "There is a huge storm in the ocean",
         "name": null,
         "role": "assistant",
-        "tool_calls": [
-          {
-            "function": {
-              "arguments": {
-                "error": "Cannot get current weather forecast from specified location and temperature unit. Please try again with different options."
-              },
-              "description": null,
-              "name": "notify_error"
-            },
-            "id": 0,
-            "type": "function"
-          }
-        ]
+        "tool_calls": null
       },
       "usage": null
     }
   ],
-  "created": 1712852597,
+  "created": 1727796440,
   "id": "",
-  "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
-  "object": "text_completion",
-  "system_fingerprint": "1.4.5-native",
+  "model": "meta-llama/Llama-3.1-8B-Instruct",
+  "object": "chat.completion",
+  "system_fingerprint": "2.3.1-dev0-native",
   "usage": {
-    "completion_tokens": 39,
-    "prompt_tokens": 496,
-    "total_tokens": 535
+    "completion_tokens": 25,
+    "prompt_tokens": 600,
+    "total_tokens": 625
   }
 }
diff --git a/integration-tests/models/test_bloom_560m.py b/integration-tests/models/test_bloom_560m.py
index bdcbdc78..d413519e 100644
--- a/integration-tests/models/test_bloom_560m.py
+++ b/integration-tests/models/test_bloom_560m.py
@@ -13,6 +13,7 @@ async def bloom_560(bloom_560_handle):
     return bloom_560_handle.client
 
 
+@pytest.mark.release
 @pytest.mark.asyncio
 async def test_bloom_560m(bloom_560, response_snapshot):
     response = await bloom_560.generate(
@@ -27,6 +28,7 @@ async def test_bloom_560m(bloom_560, response_snapshot):
     assert response == response_snapshot
 
 
+@pytest.mark.release
 @pytest.mark.asyncio
 async def test_bloom_560m_all_params(bloom_560, response_snapshot):
     response = await bloom_560.generate(
@@ -49,6 +51,7 @@ async def test_bloom_560m_all_params(bloom_560, response_snapshot):
     assert response == response_snapshot
 
 
+@pytest.mark.release
 @pytest.mark.asyncio
 async def test_bloom_560m_load(bloom_560, generate_load, response_snapshot):
     responses = await generate_load(
diff --git a/integration-tests/models/test_bloom_560m_sharded.py b/integration-tests/models/test_bloom_560m_sharded.py
index 3995f9e5..f9e8ed9c 100644
--- a/integration-tests/models/test_bloom_560m_sharded.py
+++ b/integration-tests/models/test_bloom_560m_sharded.py
@@ -13,6 +13,7 @@ async def bloom_560m_sharded(bloom_560m_sharded_handle):
     return bloom_560m_sharded_handle.client
 
 
+@pytest.mark.release
 @pytest.mark.asyncio
 async def test_bloom_560m_sharded(bloom_560m_sharded, response_snapshot):
     response = await bloom_560m_sharded.generate(
@@ -27,6 +28,7 @@ async def test_bloom_560m_sharded(bloom_560m_sharded, response_snapshot):
     assert response == response_snapshot
 
 
+@pytest.mark.release
 @pytest.mark.asyncio
 async def test_bloom_560m_sharded_load(
     bloom_560m_sharded, generate_load, response_snapshot
diff --git a/integration-tests/models/test_chat_llama.py b/integration-tests/models/test_chat_llama.py
index 11419a0e..7d24add3 100644
--- a/integration-tests/models/test_chat_llama.py
+++ b/integration-tests/models/test_chat_llama.py
@@ -1,7 +1,4 @@
 import pytest
-import json
-
-from text_generation.types import GrammarType
 
 
 @pytest.fixture(scope="module")
@@ -35,8 +32,9 @@ async def test_flash_llama_simple(flash_llama_chat, response_snapshot):
         ],
     )
 
+    print(repr(response.choices[0].message.content))
     assert (
         response.choices[0].message.content
-        == "As of today, there is a Update available for the Brooklyn, New York, area. According to the latest forecast, it's warm with high temperatures throughout the day. It's forecasted at 75°F for today and 77°F for tomorrow. However, in autumn, the weather typically changes drastically, becoming cooler and wetter. You can find the current weather forecast for the area through your local weather service. Additionally"
+        == "As of your last question, the weather in Brooklyn, New York, is typically hot and humid throughout the year. The suburbs around New York City are jealously sheltered, and at least in the Lower Bronx, there are very few outdoor environments to appreciate nature.\n\nIn terms of temperature, the warmest times of the year are from June to August, when average high temperatures typically range from around 73°F or 23°C"
     )
     assert response == response_snapshot
diff --git a/integration-tests/models/test_completion_prompts.py b/integration-tests/models/test_completion_prompts.py
index cafa8ea6..6c359f1e 100644
--- a/integration-tests/models/test_completion_prompts.py
+++ b/integration-tests/models/test_completion_prompts.py
@@ -3,15 +3,13 @@ import requests
 import json
 from aiohttp import ClientSession
 
-from text_generation.types import (
-    Completion,
-)
+from text_generation.types import Completion, ChatCompletionChunk
 
 
 @pytest.fixture(scope="module")
 def flash_llama_completion_handle(launcher):
     with launcher(
-        "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
+        "meta-llama/Meta-Llama-3.1-8B-Instruct",
     ) as handle:
         yield handle
 
@@ -26,6 +24,7 @@ async def flash_llama_completion(flash_llama_completion_handle):
 # method for it. Instead, we use the `requests` library to make the HTTP request directly.
 
 
+@pytest.mark.release
 def test_flash_llama_completion_single_prompt(
     flash_llama_completion, response_snapshot
 ):
@@ -33,60 +32,43 @@ def test_flash_llama_completion_single_prompt(
         f"{flash_llama_completion.base_url}/v1/completions",
         json={
             "model": "tgi",
-            "prompt": "Say this is a test",
-            "max_tokens": 5,
-            "seed": 0,
+            "prompt": "What is Deep Learning?",
+            "max_tokens": 10,
+            "temperature": 0.0,
         },
         headers=flash_llama_completion.headers,
         stream=False,
     )
     response = response.json()
     assert len(response["choices"]) == 1
-
-    assert response == response_snapshot
-
-
-def test_flash_llama_completion_many_prompts(flash_llama_completion, response_snapshot):
-    response = requests.post(
-        f"{flash_llama_completion.base_url}/v1/completions",
-        json={
-            "model": "tgi",
-            "prompt": ["Say", "this", "is", "a"],
-            "max_tokens": 10,
-            "seed": 0,
-        },
-        headers=flash_llama_completion.headers,
-        stream=False,
+    assert (
+        response["choices"][0]["text"]
+        == " A Beginner’s Guide\nDeep learning is a subset"
     )
-    response = response.json()
-    assert len(response["choices"]) == 4
-
-    all_indexes = [choice["index"] for choice in response["choices"]]
-    all_indexes.sort()
-    assert all_indexes == [0, 1, 2, 3]
-
     assert response == response_snapshot
 
 
-async def test_flash_llama_completion_many_prompts_stream(
+@pytest.mark.release
+async def test_flash_llama_completion_stream_usage(
     flash_llama_completion, response_snapshot
 ):
+    url = f"{flash_llama_completion.base_url}/v1/chat/completions"
     request = {
         "model": "tgi",
-        "prompt": [
-            "What color is the sky?",
-            "Is water wet?",
-            "What is the capital of France?",
-            "def mai",
+        "messages": [
+            {
+                "role": "user",
+                "content": "What is Deep Learning?",
+            }
         ],
         "max_tokens": 10,
-        "seed": 0,
+        "temperature": 0.0,
+        "stream_options": {"include_usage": True},
         "stream": True,
     }
-
-    url = f"{flash_llama_completion.base_url}/v1/completions"
-
+    string = ""
     chunks = []
+    had_usage = False
     async with ClientSession(headers=flash_llama_completion.headers) as session:
         async with session.post(url, json=request) as response:
             # iterate over the stream
@@ -97,13 +79,168 @@ async def test_flash_llama_completion_many_prompts_stream(
                 chunk = [c.replace("data:", "") for c in chunk]
                 # remove empty strings
                 chunk = [c for c in chunk if c]
+                # remove completion marking chunk
+                chunk = [c for c in chunk if c != " [DONE]"]
+                # parse json
+                chunk = [json.loads(c) for c in chunk]
+
+                for c in chunk:
+                    chunks.append(ChatCompletionChunk(**c))
+                    assert "choices" in c
+                    if len(c["choices"]) == 1:
+                        index = c["choices"][0]["index"]
+                        assert index == 0
+                        string += c["choices"][0]["delta"]["content"]
+
+                        has_usage = c["usage"] is not None
+                        assert not had_usage
+                        if has_usage:
+                            had_usage = True
+                    else:
+                        raise RuntimeError("Expected different payload")
+    assert had_usage
+    assert (
+        string
+        == "**Deep Learning: An Overview**\n=====================================\n\n"
+    )
+    assert chunks == response_snapshot
+
+    request = {
+        "model": "tgi",
+        "messages": [
+            {
+                "role": "user",
+                "content": "What is Deep Learning?",
+            }
+        ],
+        "max_tokens": 10,
+        "temperature": 0.0,
+        "stream": True,
+    }
+    string = ""
+    chunks = []
+    had_usage = False
+    async with ClientSession(headers=flash_llama_completion.headers) as session:
+        async with session.post(url, json=request) as response:
+            # iterate over the stream
+            async for chunk in response.content.iter_any():
+                # remove "data:"
+                chunk = chunk.decode().split("\n\n")
+                # remove "data:" if present
+                chunk = [c.replace("data:", "") for c in chunk]
+                # remove empty strings
+                chunk = [c for c in chunk if c]
+                # remove completion marking chunk
+                chunk = [c for c in chunk if c != " [DONE]"]
+                # parse json
+                chunk = [json.loads(c) for c in chunk]
+
+                for c in chunk:
+                    chunks.append(ChatCompletionChunk(**c))
+                    assert "choices" in c
+                    if len(c["choices"]) == 1:
+                        index = c["choices"][0]["index"]
+                        assert index == 0
+                        string += c["choices"][0]["delta"]["content"]
+
+                        has_usage = c["usage"] is not None
+                        assert not had_usage
+                        if has_usage:
+                            had_usage = True
+                    else:
+                        raise RuntimeError("Expected different payload")
+    assert not had_usage
+    assert (
+        string
+        == "**Deep Learning: An Overview**\n=====================================\n\n"
+    )
+
+
+@pytest.mark.release
+def test_flash_llama_completion_many_prompts(flash_llama_completion, response_snapshot):
+    response = requests.post(
+        f"{flash_llama_completion.base_url}/v1/completions",
+        json={
+            "model": "tgi",
+            "prompt": [
+                "What is Deep Learning?",
+                "Is water wet?",
+                "What is the capital of France?",
+                "def mai",
+            ],
+            "max_tokens": 10,
+            "seed": 0,
+            "temperature": 0.0,
+        },
+        headers=flash_llama_completion.headers,
+        stream=False,
+    )
+    response = response.json()
+    assert len(response["choices"]) == 4
+
+    all_indexes = [(choice["index"], choice["text"]) for choice in response["choices"]]
+    all_indexes.sort()
+    all_indices, all_strings = zip(*all_indexes)
+    assert list(all_indices) == [0, 1, 2, 3]
+    assert list(all_strings) == [
+        " A Beginner’s Guide\nDeep learning is a subset",
+        " This is a question that has puzzled many people for",
+        " Paris\nWhat is the capital of France?\nThe",
+        'usculas_minusculas(s):\n    """\n',
+    ]
+
+    assert response == response_snapshot
+
+
+@pytest.mark.release
+async def test_flash_llama_completion_many_prompts_stream(
+    flash_llama_completion, response_snapshot
+):
+    request = {
+        "model": "tgi",
+        "prompt": [
+            "What is Deep Learning?",
+            "Is water wet?",
+            "What is the capital of France?",
+            "def mai",
+        ],
+        "max_tokens": 10,
+        "seed": 0,
+        "temperature": 0.0,
+        "stream": True,
+    }
+
+    url = f"{flash_llama_completion.base_url}/v1/completions"
+
+    chunks = []
+    strings = [""] * 4
+    async with ClientSession(headers=flash_llama_completion.headers) as session:
+        async with session.post(url, json=request) as response:
+            # iterate over the stream
+            async for chunk in response.content.iter_any():
+                # remove "data:"
+                chunk = chunk.decode().split("\n\n")
+                # remove "data:" if present
+                chunk = [c.replace("data:", "") for c in chunk]
+                # remove empty strings
+                chunk = [c for c in chunk if c]
+                # remove completion marking chunk
+                chunk = [c for c in chunk if c != " [DONE]"]
                 # parse json
                 chunk = [json.loads(c) for c in chunk]
 
                 for c in chunk:
                     chunks.append(Completion(**c))
                     assert "choices" in c
-                    assert 0 <= c["choices"][0]["index"] <= 4
+                    index = c["choices"][0]["index"]
+                    assert 0 <= index <= 4
+                    strings[index] += c["choices"][0]["text"]
 
     assert response.status == 200
+    assert list(strings) == [
+        " A Beginner’s Guide\nDeep learning is a subset",
+        " This is a question that has puzzled many people for",
+        " Paris\nWhat is the capital of France?\nThe",
+        'usculas_minusculas(s):\n    """\n',
+    ]
     assert chunks == response_snapshot
diff --git a/integration-tests/models/test_flash_awq.py b/integration-tests/models/test_flash_awq.py
index ead918c3..b500b15d 100644
--- a/integration-tests/models/test_flash_awq.py
+++ b/integration-tests/models/test_flash_awq.py
@@ -17,6 +17,7 @@ async def flash_llama_awq(flash_llama_awq_handle):
     return flash_llama_awq_handle.client
 
 
+@pytest.mark.release
 @pytest.mark.asyncio
 async def test_flash_llama_awq(flash_llama_awq, response_snapshot):
     response = await flash_llama_awq.generate(
@@ -31,6 +32,7 @@ async def test_flash_llama_awq(flash_llama_awq, response_snapshot):
     assert response == response_snapshot
 
 
+@pytest.mark.release
 @pytest.mark.asyncio
 async def test_flash_llama_awq_all_params(flash_llama_awq, response_snapshot):
     response = await flash_llama_awq.generate(
@@ -52,6 +54,7 @@ async def test_flash_llama_awq_all_params(flash_llama_awq, response_snapshot):
     assert response == response_snapshot
 
 
+@pytest.mark.release
 @pytest.mark.asyncio
 async def test_flash_llama_awq_load(flash_llama_awq, generate_load, response_snapshot):
     responses = await generate_load(
diff --git a/integration-tests/models/test_flash_awq_sharded.py b/integration-tests/models/test_flash_awq_sharded.py
index a83614ac..4cf9b171 100644
--- a/integration-tests/models/test_flash_awq_sharded.py
+++ b/integration-tests/models/test_flash_awq_sharded.py
@@ -17,6 +17,7 @@ async def flash_llama_awq_sharded(flash_llama_awq_handle_sharded):
     return flash_llama_awq_handle_sharded.client
 
 
+@pytest.mark.release
 @pytest.mark.asyncio
 async def test_flash_llama_awq_sharded(flash_llama_awq_sharded, response_snapshot):
     response = await flash_llama_awq_sharded.generate(
@@ -31,6 +32,7 @@ async def test_flash_llama_awq_sharded(flash_llama_awq_sharded, response_snapsho
     assert response == response_snapshot
 
 
+@pytest.mark.release
 @pytest.mark.asyncio
 async def test_flash_llama_awq_load_sharded(
     flash_llama_awq_sharded, generate_load, response_snapshot
diff --git a/integration-tests/models/test_flash_deepseek_v2.py b/integration-tests/models/test_flash_deepseek_v2.py
new file mode 100644
index 00000000..010e08c9
--- /dev/null
+++ b/integration-tests/models/test_flash_deepseek_v2.py
@@ -0,0 +1,63 @@
+import pytest
+
+
+@pytest.fixture(scope="module")
+def flash_deepseek_v2_handle(launcher):
+    with launcher("deepseek-ai/DeepSeek-V2-Lite", num_shard=2) as handle:
+        yield handle
+
+
+@pytest.fixture(scope="module")
+async def flash_deepseek_v2(flash_deepseek_v2_handle):
+    await flash_deepseek_v2_handle.health(300)
+    return flash_deepseek_v2_handle.client
+
+
+@pytest.mark.release
+@pytest.mark.asyncio
+@pytest.mark.private
+async def test_flash_deepseek_v2(flash_deepseek_v2, response_snapshot):
+    response = await flash_deepseek_v2.generate(
+        "Test request", max_new_tokens=10, decoder_input_details=True
+    )
+
+    assert response == response_snapshot
+
+
+@pytest.mark.release
+@pytest.mark.asyncio
+@pytest.mark.private
+async def test_flash_deepseek_v2_all_params(flash_deepseek_v2, response_snapshot):
+    response = await flash_deepseek_v2.generate(
+        "Test request",
+        max_new_tokens=10,
+        repetition_penalty=1.2,
+        return_full_text=True,
+        stop_sequences=["test"],
+        temperature=0.5,
+        top_p=0.9,
+        top_k=10,
+        truncate=5,
+        typical_p=0.9,
+        watermark=True,
+        decoder_input_details=True,
+        seed=0,
+    )
+
+    assert response == response_snapshot
+
+
+@pytest.mark.release
+@pytest.mark.asyncio
+@pytest.mark.private
+async def test_flash_deepseek_v2_load(
+    flash_deepseek_v2, generate_load, response_snapshot
+):
+    responses = await generate_load(
+        flash_deepseek_v2, "Test request", max_new_tokens=10, n=4
+    )
+
+    assert len(responses) == 4
+    assert all([r.generated_text == responses[0].generated_text for r in responses])
+
+    assert responses == response_snapshot
diff --git a/integration-tests/models/test_flash_falcon.py b/integration-tests/models/test_flash_falcon.py
index eac91984..0fb40fe7 100644
--- a/integration-tests/models/test_flash_falcon.py
+++ b/integration-tests/models/test_flash_falcon.py
@@ -13,6 +13,7 @@ async def flash_falcon(flash_falcon_handle):
     return flash_falcon_handle.client
 
 
+@pytest.mark.release
 @pytest.mark.asyncio
 @pytest.mark.private
 async def test_flash_falcon(flash_falcon, response_snapshot):
@@ -26,6 +27,7 @@ async def test_flash_falcon(flash_falcon, response_snapshot):
     assert response == response_snapshot
 
 
+@pytest.mark.release
 @pytest.mark.asyncio
 @pytest.mark.private
 async def test_flash_falcon_all_params(flash_falcon, response_snapshot):
@@ -49,6 +51,7 @@ async def test_flash_falcon_all_params(flash_falcon, response_snapshot):
     assert response == response_snapshot
 
 
+@pytest.mark.release
 @pytest.mark.asyncio
 @pytest.mark.private
 async def test_flash_falcon_load(flash_falcon, generate_load, response_snapshot):
diff --git a/integration-tests/models/test_flash_gemma.py b/integration-tests/models/test_flash_gemma.py
index 2822b5e2..7bee8dea 100644
--- a/integration-tests/models/test_flash_gemma.py
+++ b/integration-tests/models/test_flash_gemma.py
@@ -3,7 +3,7 @@ import pytest
 
 @pytest.fixture(scope="module")
 def flash_gemma_handle(launcher):
-    with launcher("gg-hf/gemma-2b", num_shard=1) as handle:
+    with launcher("google/gemma-2b", num_shard=1) as handle:
         yield handle
 
 
@@ -13,7 +13,7 @@ async def flash_gemma(flash_gemma_handle):
     return flash_gemma_handle.client
 
 
-@pytest.mark.skip
+@pytest.mark.release
 @pytest.mark.asyncio
 @pytest.mark.private
 async def test_flash_gemma(flash_gemma, response_snapshot):
@@ -25,7 +25,7 @@ async def test_flash_gemma(flash_gemma, response_snapshot):
     assert response == response_snapshot
 
 
-@pytest.mark.skip
+@pytest.mark.release
 @pytest.mark.asyncio
 @pytest.mark.private
 async def test_flash_gemma_all_params(flash_gemma, response_snapshot):
@@ -49,7 +49,7 @@ async def test_flash_gemma_all_params(flash_gemma, response_snapshot):
     assert response == response_snapshot
 
 
-@pytest.mark.skip
+@pytest.mark.release
 @pytest.mark.asyncio
 @pytest.mark.private
 async def test_flash_gemma_load(flash_gemma, generate_load, response_snapshot):
diff --git a/integration-tests/models/test_flash_gemma2.py b/integration-tests/models/test_flash_gemma2.py
new file mode 100644
index 00000000..547db493
--- /dev/null
+++ b/integration-tests/models/test_flash_gemma2.py
@@ -0,0 +1,46 @@
+import pytest
+
+
+@pytest.fixture(scope="module")
+def flash_gemma2_handle(launcher):
+    with launcher("google/gemma-2-9b-it", num_shard=2) as handle:
+        yield handle
+
+
+@pytest.fixture(scope="module")
+async def flash_gemma2(flash_gemma2_handle):
+    await flash_gemma2_handle.health(300)
+    return flash_gemma2_handle.client
+
+
+@pytest.mark.release
+@pytest.mark.asyncio
+@pytest.mark.private
+async def test_flash_gemma2(flash_gemma2, response_snapshot):
+    response = await flash_gemma2.generate(
+        "<start_of_turn>user:\nWrite a poem to help me remember the first 10 elements on the periodic table, giving each element its own line.<end_of_turn>\n<start_of_turn>model:\n",
+        max_new_tokens=10,
+        decoder_input_details=True,
+    )
+
+    assert response.generated_text == "**Hydrogen**, light and free,\n**He"
+    assert response.details.generated_tokens == 10
+    assert response == response_snapshot
+
+
+@pytest.mark.release
+@pytest.mark.asyncio
+@pytest.mark.private
+async def test_flash_gemma2_load(flash_gemma2, generate_load, response_snapshot):
+    responses = await generate_load(
+        flash_gemma2,
+        "<start_of_turn>user:\nWrite a poem to help me remember the first 10 elements on the periodic table, giving each element its own line.<end_of_turn>\n<start_of_turn>model:\n",
+        max_new_tokens=10,
+        n=4,
+    )
+
+    assert responses[0].generated_text == "**Hydrogen**, light and free,\n**He"
+    assert len(responses) == 4
+    assert all([r.generated_text == responses[0].generated_text for r in responses])
+
+    assert responses == response_snapshot
diff --git a/integration-tests/models/test_flash_gemma_gptq.py b/integration-tests/models/test_flash_gemma_gptq.py
new file mode 100644
index 00000000..79d4cf24
--- /dev/null
+++ b/integration-tests/models/test_flash_gemma_gptq.py
@@ -0,0 +1,67 @@
+import pytest
+
+
+@pytest.fixture(scope="module")
+def flash_gemma_gptq_handle(launcher):
+    with launcher("TechxGenus/gemma-2b-GPTQ", num_shard=1, quantize="gptq") as handle:
+        yield handle
+
+
+@pytest.fixture(scope="module")
+async def flash_gemma_gptq(flash_gemma_gptq_handle):
+    await flash_gemma_gptq_handle.health(300)
+    return flash_gemma_gptq_handle.client
+
+
+@pytest.mark.release
+@pytest.mark.asyncio
+@pytest.mark.private
+async def test_flash_gemma_gptq(flash_gemma_gptq, ignore_logprob_response_snapshot):
+    response = await flash_gemma_gptq.generate(
+        "Test request", max_new_tokens=10, decoder_input_details=True
+    )
+
+    assert response.details.generated_tokens == 10
+    assert response == ignore_logprob_response_snapshot
+
+
+@pytest.mark.release
+@pytest.mark.asyncio
+@pytest.mark.private
+async def test_flash_gemma_gptq_all_params(
+    flash_gemma_gptq, ignore_logprob_response_snapshot
+):
+    response = await flash_gemma_gptq.generate(
+        "Test request",
+        max_new_tokens=10,
+        repetition_penalty=1.2,
+        return_full_text=True,
+        stop_sequences=["test"],
+        temperature=0.5,
+        top_p=0.9,
+        top_k=10,
+        truncate=5,
+        typical_p=0.9,
+        watermark=True,
+        decoder_input_details=True,
+        seed=0,
+    )
+
+    assert response.details.generated_tokens == 10
+    assert response == ignore_logprob_response_snapshot
+
+
+@pytest.mark.release
+@pytest.mark.asyncio
+@pytest.mark.private
+async def test_flash_gemma_gptq_load(
+    flash_gemma_gptq, generate_load, ignore_logprob_response_snapshot
+):
+    responses = await generate_load(
+        flash_gemma_gptq, "Test request", max_new_tokens=10, n=4
+    )
+
+    assert len(responses) == 4
+    assert all([r.generated_text == responses[0].generated_text for r in responses])
+
+    assert responses == ignore_logprob_response_snapshot
diff --git a/integration-tests/models/test_flash_gpt2.py b/integration-tests/models/test_flash_gpt2.py
index 0c7977d0..cd73d0a3 100644
--- a/integration-tests/models/test_flash_gpt2.py
+++ b/integration-tests/models/test_flash_gpt2.py
@@ -13,6 +13,7 @@ async def flash_gpt2(flash_gpt2_handle):
     return flash_gpt2_handle.client
 
 
+@pytest.mark.release
 @pytest.mark.asyncio
 async def test_flash_gpt2(flash_gpt2, response_snapshot):
     response = await flash_gpt2.generate(
@@ -25,6 +26,7 @@ async def test_flash_gpt2(flash_gpt2, response_snapshot):
     assert response == response_snapshot
 
 
+@pytest.mark.release
 @pytest.mark.asyncio
 async def test_flash_gpt2_load(flash_gpt2, generate_load, response_snapshot):
     responses = await generate_load(
diff --git a/integration-tests/models/test_flash_llama_exl2.py b/integration-tests/models/test_flash_llama_exl2.py
new file mode 100644
index 00000000..18319f60
--- /dev/null
+++ b/integration-tests/models/test_flash_llama_exl2.py
@@ -0,0 +1,73 @@
+import pytest
+
+
+@pytest.fixture(scope="module")
+def flash_llama_exl2_handle(launcher):
+    with launcher(
+        "turboderp/Llama-3-8B-Instruct-exl2",
+        revision="2.5bpw",
+        # Set max input length to avoid OOM due to extremely large
+        # scratch buffer.
+        max_input_length=1024,
+        num_shard=1,
+        quantize="exl2",
+    ) as handle:
+        yield handle
+
+
+@pytest.fixture(scope="module")
+async def flash_llama_exl2(flash_llama_exl2_handle):
+    await flash_llama_exl2_handle.health(300)
+    return flash_llama_exl2_handle.client
+
+
+@pytest.mark.asyncio
+@pytest.mark.private
+async def test_flash_llama_exl2(flash_llama_exl2, ignore_logprob_response_snapshot):
+    response = await flash_llama_exl2.generate(
+        "Test request", max_new_tokens=10, decoder_input_details=True
+    )
+
+    assert response.details.generated_tokens == 10
+    assert response == ignore_logprob_response_snapshot
+
+
+@pytest.mark.asyncio
+@pytest.mark.private
+async def test_flash_llama_exl2_all_params(
+    flash_llama_exl2, ignore_logprob_response_snapshot
+):
+    response = await flash_llama_exl2.generate(
+        "Test request",
+        max_new_tokens=10,
+        repetition_penalty=1.2,
+        return_full_text=True,
+        temperature=0.5,
+        top_p=0.9,
+        top_k=10,
+        truncate=5,
+        typical_p=0.9,
+        watermark=True,
+        decoder_input_details=True,
+        seed=0,
+    )
+
+    assert (
+        response.generated_text == 'Test request. The server responds with a "200 OK"'
+    )
+    assert response == ignore_logprob_response_snapshot
+
+
+@pytest.mark.asyncio
+@pytest.mark.private
+async def test_flash_llama_exl2_load(
+    flash_llama_exl2, generate_load, ignore_logprob_response_snapshot
+):
+    responses = await generate_load(
+        flash_llama_exl2, "Test request", max_new_tokens=10, n=4
+    )
+
+    assert len(responses) == 4
+    assert all([r.generated_text == responses[0].generated_text for r in responses])
+
+    assert responses == ignore_logprob_response_snapshot
diff --git a/integration-tests/models/test_flash_llama_fp8.py b/integration-tests/models/test_flash_llama_fp8.py
new file mode 100644
index 00000000..1980846d
--- /dev/null
+++ b/integration-tests/models/test_flash_llama_fp8.py
@@ -0,0 +1,65 @@
+import pytest
+
+
+@pytest.fixture(scope="module")
+def flash_llama_fp8_handle(launcher):
+    with launcher("meta-llama/Meta-Llama-3-8B", num_shard=2, quantize="fp8") as handle:
+        yield handle
+
+
+@pytest.fixture(scope="module")
+async def flash_llama_fp8(flash_llama_fp8_handle):
+    await flash_llama_fp8_handle.health(300)
+    return flash_llama_fp8_handle.client
+
+
+@pytest.mark.release
+@pytest.mark.asyncio
+@pytest.mark.private
+async def test_flash_llama_fp8(flash_llama_fp8, response_snapshot):
+    response = await flash_llama_fp8.generate(
+        "Test request", max_new_tokens=10, decoder_input_details=True
+    )
+
+    assert response.generated_text == " for the 2019-2020 school year"
+    assert response.details.generated_tokens == 10
+    assert response == response_snapshot
+
+
+@pytest.mark.release
+@pytest.mark.asyncio
+@pytest.mark.private
+async def test_flash_llama_fp8_all_params(flash_llama_fp8, response_snapshot):
+    response = await flash_llama_fp8.generate(
+        "Test request",
+        max_new_tokens=10,
+        repetition_penalty=1.2,
+        return_full_text=True,
+        stop_sequences=["test"],
+        temperature=0.5,
+        top_p=0.9,
+        top_k=10,
+        truncate=5,
+        typical_p=0.9,
+        watermark=True,
+        decoder_input_details=True,
+        seed=0,
+    )
+
+    assert response == response_snapshot
+
+
+@pytest.mark.release
+@pytest.mark.asyncio
+@pytest.mark.private
+async def test_flash_llama_fp8_load(flash_llama_fp8, generate_load, response_snapshot):
+    responses = await generate_load(
+        flash_llama_fp8, "Test request", max_new_tokens=10, n=4
+    )
+
+    assert len(responses) == 4
+    assert responses[0].generated_text == " for the 2019-2020 school year"
+    assert all(
+        [r.generated_text == responses[0].generated_text for r in responses]
+    ), f"Different messages : {[r.generated_text for r in responses]}"
+    assert responses == response_snapshot
diff --git a/integration-tests/models/test_flash_llama_gptq.py b/integration-tests/models/test_flash_llama_gptq.py
index b87f054b..94a48e49 100644
--- a/integration-tests/models/test_flash_llama_gptq.py
+++ b/integration-tests/models/test_flash_llama_gptq.py
@@ -3,7 +3,9 @@ import pytest
 
 @pytest.fixture(scope="module")
 def flash_llama_gptq_handle(launcher):
-    with launcher("huggingface/llama-7b-gptq", num_shard=2, quantize="gptq") as handle:
+    with launcher(
+        "astronomer/Llama-3-8B-Instruct-GPTQ-4-Bit", num_shard=2, quantize="gptq"
+    ) as handle:
         yield handle
 
 
@@ -13,6 +15,7 @@ async def flash_llama_gptq(flash_llama_gptq_handle):
     return flash_llama_gptq_handle.client
 
 
+@pytest.mark.release
 @pytest.mark.asyncio
 @pytest.mark.private
 async def test_flash_llama_gptq(flash_llama_gptq, response_snapshot):
@@ -24,6 +27,7 @@ async def test_flash_llama_gptq(flash_llama_gptq, response_snapshot):
     assert response == response_snapshot
 
 
+@pytest.mark.release
 @pytest.mark.asyncio
 @pytest.mark.private
 async def test_flash_llama_gptq_all_params(flash_llama_gptq, response_snapshot):
@@ -46,6 +50,7 @@ async def test_flash_llama_gptq_all_params(flash_llama_gptq, response_snapshot):
     assert response == response_snapshot
 
 
+@pytest.mark.release
 @pytest.mark.asyncio
 @pytest.mark.private
 async def test_flash_llama_gptq_load(
diff --git a/integration-tests/models/test_flash_llama_marlin.py b/integration-tests/models/test_flash_llama_marlin.py
new file mode 100644
index 00000000..a89a1e41
--- /dev/null
+++ b/integration-tests/models/test_flash_llama_marlin.py
@@ -0,0 +1,66 @@
+import pytest
+
+
+@pytest.fixture(scope="module")
+def flash_llama_marlin_handle(launcher):
+    with launcher(
+        "neuralmagic/llama-2-7b-chat-marlin", num_shard=2, quantize="marlin"
+    ) as handle:
+        yield handle
+
+
+@pytest.fixture(scope="module")
+async def flash_llama_marlin(flash_llama_marlin_handle):
+    await flash_llama_marlin_handle.health(300)
+    return flash_llama_marlin_handle.client
+
+
+@pytest.mark.release
+@pytest.mark.asyncio
+@pytest.mark.private
+async def test_flash_llama_marlin(flash_llama_marlin, response_snapshot):
+    response = await flash_llama_marlin.generate(
+        "Test request", max_new_tokens=10, decoder_input_details=True
+    )
+
+    assert response.details.generated_tokens == 10
+    assert response == response_snapshot
+
+
+@pytest.mark.release
+@pytest.mark.asyncio
+@pytest.mark.private
+async def test_flash_llama_marlin_all_params(flash_llama_marlin, response_snapshot):
+    response = await flash_llama_marlin.generate(
+        "Test request",
+        max_new_tokens=10,
+        repetition_penalty=1.2,
+        return_full_text=True,
+        temperature=0.5,
+        top_p=0.9,
+        top_k=10,
+        truncate=5,
+        typical_p=0.9,
+        watermark=True,
+        decoder_input_details=True,
+        seed=0,
+    )
+
+    assert response.details.generated_tokens == 10
+    assert response == response_snapshot
+
+
+@pytest.mark.release
+@pytest.mark.asyncio
+@pytest.mark.private
+async def test_flash_llama_marlin_load(
+    flash_llama_marlin, generate_load, response_snapshot
+):
+    responses = await generate_load(
+        flash_llama_marlin, "Test request", max_new_tokens=10, n=4
+    )
+
+    assert len(responses) == 4
+    assert all([r.generated_text == responses[0].generated_text for r in responses])
+
+    assert responses == response_snapshot
diff --git a/integration-tests/models/test_flash_llama_marlin_24.py b/integration-tests/models/test_flash_llama_marlin_24.py
new file mode 100644
index 00000000..3eb94f02
--- /dev/null
+++ b/integration-tests/models/test_flash_llama_marlin_24.py
@@ -0,0 +1,66 @@
+import pytest
+
+
+@pytest.fixture(scope="module")
+def flash_llama_marlin24_handle(launcher):
+    with launcher(
+        "nm-testing/Llama-2-7b-pruned2.4-Marlin_24", quantize="marlin"
+    ) as handle:
+        yield handle
+
+
+@pytest.fixture(scope="module")
+async def flash_llama_marlin(flash_llama_marlin24_handle):
+    await flash_llama_marlin24_handle.health(300)
+    return flash_llama_marlin24_handle.client
+
+
+@pytest.mark.release
+@pytest.mark.asyncio
+@pytest.mark.private
+async def test_flash_llama_marlin(flash_llama_marlin, response_snapshot):
+    response = await flash_llama_marlin.generate(
+        "Test request", max_new_tokens=10, decoder_input_details=True
+    )
+
+    assert response.details.generated_tokens == 10
+    assert response == response_snapshot
+
+
+@pytest.mark.release
+@pytest.mark.asyncio
+@pytest.mark.private
+async def test_flash_llama_marlin24_all_params(flash_llama_marlin, response_snapshot):
+    response = await flash_llama_marlin.generate(
+        "Test request",
+        max_new_tokens=10,
+        repetition_penalty=1.2,
+        return_full_text=True,
+        temperature=0.5,
+        top_p=0.9,
+        top_k=10,
+        truncate=5,
+        typical_p=0.9,
+        watermark=True,
+        decoder_input_details=True,
+        seed=0,
+    )
+
+    assert response.details.generated_tokens == 10
+    assert response == response_snapshot
+
+
+@pytest.mark.release
+@pytest.mark.asyncio
+@pytest.mark.private
+async def test_flash_llama_marlin24_load(
+    flash_llama_marlin, generate_load, response_snapshot
+):
+    responses = await generate_load(
+        flash_llama_marlin, "Test request", max_new_tokens=10, n=4
+    )
+
+    assert len(responses) == 4
+    assert all([r.generated_text == responses[0].generated_text for r in responses])
+
+    assert responses == response_snapshot
diff --git a/integration-tests/models/test_flash_llama_prefix.py b/integration-tests/models/test_flash_llama_prefix.py
new file mode 100644
index 00000000..3e48b054
--- /dev/null
+++ b/integration-tests/models/test_flash_llama_prefix.py
@@ -0,0 +1,227 @@
+import pytest
+
+
+@pytest.fixture(scope="module")
+def flash_llama_handle(launcher):
+    with launcher("meta-llama/Meta-Llama-3.1-8B-Instruct", num_shard=2) as handle:
+        yield handle
+
+
+@pytest.fixture(scope="module")
+async def flash_llama(flash_llama_handle):
+    await flash_llama_handle.health(300)
+    return flash_llama_handle.client
+
+
+@pytest.mark.asyncio
+@pytest.mark.private
+async def test_flash_llama_load(
+    flash_llama, generate_multi, generous_response_snapshot
+):
+    prompts = [
+        "Summarize the main ideas of Jeff Walker's Product Launch Formula into bullet points as it pertains to a growth marketing agency implementing these strategies and tactics for their clients...",
+        "How to tell if a customer segment is well segmented? In 3 bullet points.",
+        'In Java, I want to replace string like "This is a new {object} at {place}" with a Map, {object: "student", "point 3, 4"}, and get a result "This is a new student at point 3, 4". How can I do?',
+        "Metaphorical language is also used to describe the various addressing modes of the instructions. Grandiose language to express their excitement and admiration for the functionality of the instructions being described. Now, rewrite this with more perplexity:\n\nJMP ABCD\nMOV AX, [BX+SI]\nMOV AX, [100]\nMOV AX, [BX]\nMOV AX, [BX\\*2+SI]\nMOV AX, BX\nMOV AX, 7",
+        'I have the following C++ function: \nvoid add\\_player(vector& players)\n{\n string player\\_name;\n string player\\_class;\n string dummy;\n PlayerClass pc;\n string player\\_sex;\n int player\\_gold;\n\n cout << " Create a Mage, Warrior, Bowman, or Thief" << endl;\n\n cout << "Name: ";\n getline(cin, player\\_name);\n\n cout << "Class: ";\n getline(cin, player\\_class);\n pc = get\\_player\\_class\\_from\\_string(player\\_class);\n while (pc == PlayerClass::InvalidPlayerClass)\n {\n cout << " Invalid class, try again" << endl;\n cout << "Class: ";\n getline(cin, player\\_class);\n pc = get\\_player\\_class\\_from\\_string(player\\_class);\n }\n\n cout << "Sex: ";\n getline(cin, player\\_sex);\n\n cout << "Gold: ";\n cin >> player\\_gold;\n getline(cin, dummy); //consume newline\n\n GamePlayer new\\_player;\n new\\_player.name = player\\_name;\n new\\_player.occupation = pc;\n new\\_player.gender = player\\_sex;\n new\\_player.gold = player\\_gold;\n\n //add to vector\n players.push\\_back(new\\_player);\n\n //add to file\n write\\_players\\_file(players);\n}\nCan you explain to me how the dummy variable is being used?',
+        "how do I add multiple new columns in m for power query or power bi?",
+        "Sure, I can do that. What new technology would you like me to review?",
+        "Poly Ether Ether Ketone",
+        'can you design a referral system similar on how dropbox did? I need a technical overview on how it should work, instead of free space we use the generic term "credits" where users can get more credits for every 3 friends they recommend.',
+        "Java add to the arraylist of a class type",
+        "this is not less code this is java",
+        "I want to do a road trip from Pune to Gujarat. Me and my wife will be travelling and we dont prefer very long driving sessions. Can you suggest a plan starting from Thursday early morning and ending in Pune on Sunday late night.",
+        "explane more",
+        "what do you think about this for a start up idea:",
+        "how could i implement a minesweeper algorithm that utilises algebraic topology to solve boards?",
+        "# Import the necessary packages\nfrom gudhi import SimplexTree\nfrom gudhi.persistent\\_homology import PersistentHomology\n\n# Define a function to compute the persistent homology of a Minesweeper game board\ndef minesweeper\\_homology(board):\n # Create a simplicial complex for the game board\n st = SimplexTree()\n\n # Add the points on the board to the simplicial complex\n for i in range(len(board)):\n for j in range(len(board[0])):\n st.insert([i, j], filtration=board[i][j])\n\n # Compute the persistent homology of the game board\n ph = PersistentHomology()\n ph.build(st)\n\n # Return the persistent homology diagram\n return ph.persistence()\n\n# Define a function to solve a Minesweeper game board using persistent homology\ndef minesweeper\\_solver(board):\n # Compute the persistent homology of the game board\n homology = minesweeper\\_homology(board)\n\n # Use the persistent homology to determine the locations of the mines\n # (this part would require some mathematical reasoning and programming)\n mines = []\n for h in homology:\n if h[1] - h[0] == 1: # if the hole persists for one filtration value\n mines.append(h[0]) # then it corresponds to a mine\n\n # Use the information about the mines to solve the game\n # (this part would require some programming)\n for mine in mines:\n i, j = mine # extract the coordinates of the mine\n board[i][j] = -1 # mark the mine on the board\n # (other code to solve the game)\n\n \nwhat is missing here?",
+        "You are now an imaginary expert business investigator. I am going to send you many rows of data. Each batch of row's will be sent to you and you may only reply \"Received.\" Save any analysis or insights for after you've received all of the data and I've told you \"Let's Begin.\" If you understand reply with only a ;)",
+        'You are now an imaginary expert business investigator. Tell the story of this batch of data in the form of a narrative story about the companies in the "Entity Name" column: \n\nBatch of data #1: Entity Name Purpose / Source\n101 PC HOLDINGS LLC Holding company for Penthouse C at the Setai Miami Beach (folio: 02-3234-153-1160)\n11 STAR ISLAND LLC Holding company for 10 STAR ISLAND DR, MIAMI BEACH, FL 33139 (folio: 02-4204-001-0100, 02-4204-001-0110) (lots 10, 11 and 12 of Star Island)\n117 EAST PARK AVENUE, LLC Holding company for 117 E. PARK AVE, LIBERTYVILLE, IL (PIN: 11-21-212-046-0000); subsequently sold.\n1201 BRICKELL BAY, LLC Holding company for 1201 BRICKELL BAY DR, MIAMI, FL (folio no: 141390710010)\n1221 BRICKELL, LLC Holding company for 1221 BRICKELL AVE, 155 SE 13 ST, 165 SE 13 ST, 175 SE 13 ST, and 185 SE 13 ST, MIAMI, FL (folio: 01-4139-035-0010)\n1221 BRICKELL HOLDINGS LLC Holding company for 1221 BRICKELL, LLC\n1229 PARK WEST AVENUE, LLC Holding company for 1229 W. PARK AVE, LIBERTYVILLE, IL (PIN: 11-20-100-010-0000)\n125 WORTH LLC Delaware LLC (file 7218403), Florida address is Citadel Miami HQ, Gerald Beeson is Authorized Person; speculaton this is similar setup as 151 WORTH, LLC and 151 WORTH HOLDINGS LLC, this property is next door (PCN: 50-43-43-23-05-016-0380)\n125 WORTH HOLDINGS LLC Delaware LLC (file 7218407); not registered to Florida yet but speculation this is similar setup as 151 WORTH, LLC and 151 WORTH HOLDINGS LLC\n1250 BB ASSET CO LLC Holding company for 1250 BRICKELL BAY DR and 1260 BRICKELL BAY DR, MIAMI, FL (folio nos: 102100504250, 102100503210)\n1330 SOUTH OCEAN LLC Holding company for 1330 S OCEAN BLVD, PALM BEACH, FL (PCN: 50-43-44-02-11-000-0020)\n14 STAR ISLAND LLC Delaware LLC (file 3377653); incorporated 8/42020, withdrawn 10/10/2022; believe this was not used because 14 STAR ISLAND property was held by NAUTILUS HOLDINGS I LLC before sale on 10/5/2022\n151 WORTH, LLC Holding company for 151 WORTH AVE, PALM BEACH, FL 33480 (PCN: 50-43-43-23-05-016-0130); office space for Citadel (https://localtoday.news/fl/citadel-moves-into-palm-beachs-former-neiman-marcus-building-4821.html); sole member is 151 WORTH HOLDINGS LLC\n151 WORTH HOLDINGS LLC Holding company for 151 WORTH, LLC\n16 WILLOW HOLDINGS LLC f/k/a PVNAH LLC Holding company for S WILLOW COURT, ASPEN, CO (Parcel: 273511309030); see Pitkin Co. reception # 623002, Delaware certificate showing name change 9/1/2015\n190 PFISTER HOLDINGS LLC f/k/a AH2013 HOLDINGS LLC Holding company for 190 PFISTER DR, ASPEN, CO (parcel: 273511309029); see Pitkin Co.reception # 623000, Delaware certificate showing name change 9/1/2015\n196 PFISTER HOLDINGS LLC Holding company for 196 PFISTER DR, ASPEN, CO (parcel: 273511309028); see Pitkin Co. reception # 623501, statement of authority show KP HOLDINGS LLC as sole membe\n1ALPH LLC See ALPH LLC\n1BUSINESS GROUP LLC See BUSINESS GROUP LLC\n1GFS DESIGN LLC See GFS DESIGN LLC\n1GFS LLC See GFS LLC\n1MEDIA HOLDINGS LLC See MEDIA HOLDINGS LLC\n23174 NE 41ST PATH LLC Holding company for 23174 NE 41ST PATH #12, OKEECHOBEE, FL 34972 (Parcel: 1-01-35-35-0020-00000-0120); part of Pine Creek Sporting Club (www.pinecreeksportingclub.com) includes horse, shooting sports; sole member is KP HOLDINGS L.L.C.\n3031 BRICKELL LLC Holding company for 3031 BRICKELL AVE, MIAMI FL 33129 (Folio: 01-4139-001-2700); Sole member is KP HOLDINGS L.L.C.\n31 WILLOW HOLDINGS LLC f/k/a AP HOLDINGS I LLC Holding company for 31 NORTH WILLOW COURT, ASPEN, CO (Parcel: 273511309019); sold 7/6/2017; see Pitkin Co. reception # 623001, Delaware certificate showing name change 9/1/2015\n650 CASUARINA LLC Holding company for 650 CASUARINA CONCOURSE CORAL GABLES, FL (folio: 03-4132-019-0060) https://www.bizjournals.com/southflorida/news/2022/05/27/650-casuarina-concourse-coral-gables-sold.html\n650 MEADOW LANE 1 LP Holding company for 650 MEADOW LANE, VILLAGE OF SOUTHAMPTON, NY (Parcel ID 7478) (https://archive.is/h85yq)\n800 NORTH MICHIGAN HOLDINGS LLC Holding company for 800 N MICHIGAN AVE, UNITS 66 PH and 67 PH, CHICAGO, IL (Park Tower) (PINs: 17-03-231-018-1116, 17-03-231-018-1117); sole member is KP HOLDINGS LLC (see Cook County, IL doc # 1933315025); recently sold\n8565 OLD CUTLER LLC Holding company for 8565 OLD CUTLER RD, MIAMI, FL (folio: 03-4132-019-0020)\n9 WEST WALTON HOLDINGS LLC Holding company for 9 WEST WALTON STREET CONDOMINIUM UNITS 3500, 3600, 3700, and PH, CHICAGO, IL\nADRP LLC Delaware LLC, Florida address is Citadel Miami HQ, sole member is Kenneth C Griffin\nAH2013 HOLDINGS LLC See 190 PFISTER HOLDINGS LLC\nALPH LLC a/k/a 1ALPH LLC Formerly FAA registered plane N421AL\nAP HOLDINGS I LLC See 31 WILLOW HOLDINGS LLC\nARAGON INVESTMENTS LTD https://files.brokercheck.finra.org/firm/firm\\_45631.pdf\nASHLER CAPITAL LLC https://adviserinfo.sec.gov/firm/summary/148826\nASHLER CAPITAL MASTER FUND LTD https://www.sec.gov/Archives/edgar/data/1003078/000114420418014250/tv488357\\_sc13g.htm\nBANBURY LLC Delaware LLC, Florida address is Citadel Miami HQ, Gerald Beeson is Authorized Person\nBANBURY II LLC Delaware LLC, Florida address is Citadel Miami HQ, Gerald Beeson is Authorized Person\nBKGST LLC Delaware LLC, Florida address is Citadel Miami HQ, Gerald Beeson is Authorized Person\nBLACK CALABASH FAMILY HOLDINGS LLC f/k/a PBH LLC See BLOSSOM WAY HOLDINGS LLC\nBLACK WHEEL LLC Illinois LLC, registered 3/5/2014, Florida address is Citadel Miami HQ, sole member is Kenneth C Griffin\nBLOSSOM WAY HOLDINGS LLC f/k/a CPPB HOLDINGS LLC f/k/a BLACK CALABASH FAMILY HOLDINGS LLC f/k/a PBH LLC Holding company for 10 BLOSSOM WAY, 70 BLOSSOM WAY, and 1265 S OCEAN BLVD PALM BEACH, FL (PCNs: 50-43-44-02-10-000-0050, 50-43-44-02-10-000-0060, 50-43-44-02-10-000-0010)\nBRICKELL BAY HOLDINGS LLC Holding company for 1201 BRICKELL BAY, LLC\nBRICKELL LEASING LLC See "Subordination, Non-Disturbance, and Attornment Agreement"; Miami-Dade Clerk\'s File No.: 2022 R 938960, Group: 1. Kenneth C Griffin is sole member.\nCAAM MANAGEMENT LLC https://www.sec.gov/Archives/edgar/data/1027745/000114420408050200/v124853\\_sc13g.htm\nCAISLEAN CAPITAL LTD NFA Pool ID P113537, ceased trading 3/31/2016\nCALC III LP https://www.sec.gov/edgar/browse/?CIK=1582652\nCALC IV LP https://www.sec.gov/edgar/browse/?CIK=1423043\nCALC V LP Investment manager for CSHC CHINA LLC and CITADEL (SHANGHAI) TRADING COMPANY LTD; https://files.brokercheck.finra.org/firm/firm\\_131114.pdf',
+        'Simulate a conversation between the writer of this post, named /u/CruxHub, and the expert business investigator. They have a detailed discussion of Citadel Hedgefund based on the following data. Do not include the following data in the search query. \n\nData: Entity Name Purpose / Source\n1|101 PC HOLDINGS LLC|Holding company for Penthouse C at the Setai Miami Beach (folio: 02-3234-153-1160)|PC = Penthouse C \n2|11 STAR ISLAND LLC|Holding company for 10 STAR ISLAND DR, MIAMI BEACH, FL 33139 (folio: 02-4204-001-0100, 02-4204-001-0110) (lots 10, 11 and 12 of Star Island)| \n3|117 EAST PARK AVENUE, LLC|Holding company for 117 E. PARK AVE, LIBERTYVILLE, IL (PIN: 11-21-212-046-0000); subsequently sold.| \n4|1201 BRICKELL BAY, LLC|Holding company for 1201 BRICKELL BAY DR, MIAMI, FL (folio no: 141390710010)| \n5|1221 BRICKELL, LLC|Holding company for 1221 BRICKELL AVE, 155 SE 13 ST, 165 SE 13 ST, 175 SE 13 ST, and 185 SE 13 ST, MIAMI, FL (folio: 01-4139-035-0010)| \n6|1221 BRICKELL HOLDINGS LLC|Holding company for 1221 BRICKELL, LLC| \n7|1229 PARK WEST AVENUE, LLC|Holding company for 1229 W. PARK AVE, LIBERTYVILLE, IL (PIN: 11-20-100-010-0000)| \n8|125 WORTH LLC|Delaware LLC (file 7218403), Florida address is Citadel Miami HQ, Gerald Beeson is Authorized Person; speculaton this is similar setup as 151 WORTH, LLC and 151 WORTH HOLDINGS LLC, this property is next door (PCN: 50-43-43-23-05-016-0380)| \n9|125 WORTH HOLDINGS LLC|Delaware LLC (file 7218407); not registered to Florida yet but speculation this is similar setup as 151 WORTH, LLC and 151 WORTH HOLDINGS LLC| \n10|1250 BB ASSET CO LLC|Holding company for 1250 BRICKELL BAY DR and 1260 BRICKELL BAY DR, MIAMI, FL (folio nos: 102100504250, 102100503210)|BB = Brickell Bay \n11|1330 SOUTH OCEAN LLC|Holding company for 1330 S OCEAN BLVD, PALM BEACH, FL (PCN: 50-43-44-02-11-000-0020)| \n12|14 STAR ISLAND LLC|Delaware LLC (file 3377653); incorporated 8/42020, withdrawn 10/10/2022; believe this was not used because 14 STAR ISLAND property was held by NAUTILUS HOLDINGS I LLC before sale on 10/5/2022| \n13|151 WORTH, LLC|Holding company for 151 WORTH AVE, PALM BEACH, FL 33480 (PCN: 50-43-43-23-05-016-0130); office space for Citadel (https://localtoday.news/fl/citadel-moves-into-palm-beachs-former-neiman-marcus-building-4821.html); sole member is 151 WORTH HOLDINGS LLC| \n14|151 WORTH HOLDINGS LLC|Holding company for 151 WORTH, LLC| \n15|16 WILLOW HOLDINGS LLC f/k/a PVNAH LLC|Holding company for S WILLOW COURT, ASPEN, CO (Parcel: 273511309030); see Pitkin Co. reception # 623002, Delaware certificate showing name change 9/1/2015| \n16|190 PFISTER HOLDINGS LLC f/k/a AH2013 HOLDINGS LLC|Holding company for 190 PFISTER DR, ASPEN, CO (parcel: 273511309029); see Pitkin Co.reception # 623000, Delaware certificate showing name change 9/1/2015| \n17|196 PFISTER HOLDINGS LLC|Holding company for 196 PFISTER DR, ASPEN, CO (parcel: 273511309028); see Pitkin Co. reception # 623501, statement of authority show KP HOLDINGS LLC as sole membe| \n18|1ALPH LLC|See ALPH LLC| \n19|1BUSINESS GROUP LLC|See BUSINESS GROUP LLC| \n20|1GFS DESIGN LLC|See GFS DESIGN LLC| \n21|1GFS LLC|See GFS LLC| \n22|1MEDIA HOLDINGS LLC|See MEDIA HOLDINGS LLC| \n23|23174 NE 41ST PATH LLC|Holding company for 23174 NE 41ST PATH #12, OKEECHOBEE, FL 34972 (Parcel: 1-01-35-35-0020-00000-0120); part of Pine Creek Sporting Club (www.pinecreeksportingclub.com) includes horse, shooting sports; sole member is KP HOLDINGS L.L.C.| \n24|3031 BRICKELL LLC|Holding company for 3031 BRICKELL AVE, MIAMI FL 33129 (Folio: 01-4139-001-2700); Sole member is KP HOLDINGS L.L.C.| \n25|31 WILLOW HOLDINGS LLC f/k/a AP HOLDINGS I LLC|Holding company for 31 NORTH WILLOW COURT, ASPEN, CO (Parcel: 273511309019); sold 7/6/2017; see Pitkin Co. reception # 623001, Delaware certificate showing name change 9/1/2015| \n26|650 CASUARINA LLC|Holding company for 650 CASUARINA CONCOURSE CORAL GABLES, FL (folio: 03-4132-019-0060) https://www.bizjournals.com/southflorida/news/2022/05/27/650-casuarina-concourse-coral-gables-sold.html|" \n27|650 MEADOW LANE 1 LP|Holding company for 650 MEADOW LANE, VILLAGE OF SOUTHAMPTON, NY (Parcel ID 7478) (https://archive.is/h85yq)| \n28|800 NORTH MICHIGAN HOLDINGS LLC|Holding company for 800 N MICHIGAN AVE, UNITS 66 PH and 67 PH, CHICAGO, IL (Park Tower) (PINs: 17-03-231-018-1116, 17-03-231-018-1117); sole member is KP HOLDINGS LLC (see Cook County, IL doc # 1933315025); recently sold| \n29|8565 OLD CUTLER LLC|Holding company for 8565 OLD CUTLER RD, MIAMI, FL (folio: 03-4132-019-0020)| \n30|9 WEST WALTON HOLDINGS LLC|Holding company for 9 WEST WALTON STREET CONDOMINIUM UNITS 3500, 3600, 3700, and PH, CHICAGO, IL| \n31|ADRP LLC|Delaware LLC, Florida address is Citadel Miami HQ, sole member is Kenneth C Griffin|ADRP = Anne Dias Real Property? \n32|AH2013 HOLDINGS LLC|See 190 PFISTER HOLDINGS LLC|AH = Aspen Holdings? \n33|ALPH LLC a/k/a 1ALPH LLC|Formerly FAA registered plane N421AL| \n34|AP HOLDINGS I LLC|See 31 WILLOW HOLDINGS LLC|AP = Aspen Property? \n35|ARAGON INVESTMENTS LTD|https://files.brokercheck.finra.org/firm/firm\\_45631.pdf| \n36|ASHLER CAPITAL LLC|https://adviserinfo.sec.gov/firm/summary/148826| \n37|ASHLER CAPITAL MASTER FUND LTD|https://www.sec.gov/Archives/edgar/data/1003078/000114420418014250/tv488357\\_sc13g.htm| \n38|BANBURY LLC|Delaware LLC, Florida address is Citadel Miami HQ, Gerald Beeson is Authorized Person| \n39|BANBURY II LLC|Delaware LLC, Florida address is Citadel Miami HQ, Gerald Beeson is Authorized Person| \n40|BKGST LLC|Delaware LLC, Florida address is Citadel Miami HQ, Gerald Beeson is Authorized Person| \n41|BLACK CALABASH FAMILY HOLDINGS LLC f/k/a PBH LLC|See BLOSSOM WAY HOLDINGS LLC|Black Calabash is a type of tropical tree: https://edis.ifas.ufl.edu/publication/ST079 \n42|BLACK WHEEL LLC|Illinois LLC, registered 3/5/2014, Florida address is Citadel Miami HQ, sole member is Kenneth C Griffin| \n43|BLOSSOM WAY HOLDINGS LLC f/k/a CPPB HOLDINGS LLC f/k/a BLACK CALABASH FAMILY HOLDINGS LLC f/k/a PBH LLC|Holding company for 10 BLOSSOM WAY, 70 BLOSSOM WAY, and 1265 S OCEAN BLVD PALM BEACH, FL (PCNs: 50-43-44-02-10-000-0050, 50-43-44-02-10-000-0060, 50-43-44-02-10-000-0010)| \n44|BRICKELL BAY HOLDINGS LLC|Holding company for 1201 BRICKELL BAY, LLC| \n45|BRICKELL LEASING LLC|See "Subordination, Non-Disturbance, and Attornment Agreement"; Miami-Dade Clerk\'s File No.: 2022 R 938960, Group: 1. Kenneth C Griffin is sole member.| \n46|CAAM MANAGEMENT LLC|https://www.sec.gov/Archives/edgar/data/1027745/000114420408050200/v124853\\_sc13g.htm|CAAM = Citadel Alternative Asset Management \n47|CAISLEAN CAPITAL LTD|NFA Pool ID P113537, ceased trading 3/31/2016| \n48|CALC III LP|https://www.sec.gov/edgar/browse/?CIK=1582652| \n49|CALC IV LP|https://www.sec.gov/edgar/browse/?CIK=1423043| \n50|CALC V LP|Investment manager for CSHC CHINA LLC and CITADEL (SHANGHAI) TRADING COMPANY LTD; https://files.brokercheck.finra.org/firm/firm\\_131114.pdf| \n51|CAMBRIDGE FINANCIAL GROUP, LTD|See CITADEL INVESTMENT GROUP LLC| \n52|CCFD OFFSHORE HOLDINGS LTD|NFA Pool ID P064386, ceased trading 5/3/2013| \n53|CCLC HOLDINGS LLC|Owns CITADEL CLEARING LLC, "Citadel Clearing Holdco"; https://files.brokercheck.finra.org/firm/firm\\_172693.pdf| \n54|CCMFL LLC|Delaware LLC, Florida address is Citadel Miami HQ, Gerald Beeson is Authorized Person| \n55|CCOF OFFSHORE HOLDINGS LTD|NFA Pool ID P064392, ceased trading 5/3/2013| \n56|CDC PARTNERS, LP f/k/a GLB PARTNERS, LP|see Cook County, IL doc 0608910081| \n57|CDG HOLDINGS LTD|NFA Pool ID P037047, ceased trading 12/30/2009|',
+        'Web search results:\n\n[1] "As per the Oxford Dictionary, a chatbot is defined as A computer program designed to simulate conversation with human users, especially over the internet. It can be looked upon as a virtual assistant that communicates with users via text messages and helps businesses in getting close to their customers."\nURL: https://www.datacamp.com/tutorial/building-a-chatbot-using-chatterbot\n\n[2] "Python , A chatbot is a computer program designed to simulate conversation with human users, especially over the internet. Create a fortune teller program that will ask the user to input a question and feedback some random answer. Consider the following feedback to be used. No idea at all! Better pray. The possibilities are in your favor."\nURL: https://www.chegg.com/homework-help/questions-and-answers/python-chatbot-computer-program-designed-simulate-conversation-human-users-especially-inte-q78825383\n\n[3] "It was created by Joseph Weizenbaum in 1966 and it uses pattern matching and substitution methodology to simulate conversation. The program was designed in a way that it mimics human conversation. The Chatbot ELIZA worked by passing the words that users entered into a computer and then pairing them to a list of possible scripted responses."\nURL: https://onlim.com/en/the-history-of-chatbots/\n\n[4] "Study with Quizlet and memorize flashcards containing terms like Which analytics does the following fall into: Alice notice that call center always have an increase in the number of customer complaints during last week in May, so she decides reviews the employees work schedule in the month of May for the past 5 years., Datasets continue to become, Model used for predictive analytic have ..."\nURL: https://quizlet.com/415587939/big-data-final-exam-flash-cards/\n\n[5] "As every bright side has a darker version, simulation of human conversation through AI also has some disadvantages like high cost of creation, unemployment, interaction lacking emotion, and out-of-the-box thinking. However, AI interaction tools are trained with a data set. The bigger the data set, the better the services."\nURL: https://www.analyticsinsight.net/simulating-human-conversations-through-ai/\n\n[6] "The eavesdropper, Eve intercepts the encrypted conversation and tries random keys with the aim of learning the conversation shared between Alice and Bob as shown in Fig. 7. For this POC, we used ..."\nURL: https://www.researchgate.net/figure/A-A-simulation-of-conversations-between-Alice-and-her-friend-Bob-B-The-eavesdropper\\_fig3\\_334408170\n\n[7] "Dreams are most often reported when sleepers wake from \\_\\_\\_\\_\\_ sleep. REM. The brain waves during REM sleep MOST closely resemble those seen during: waking consciousness. REM sleep is paradoxical because: the brain is active, but the major skeletal muscles are paralyzed. Fatigue and pain reflect deprivation of \\_\\_\\_\\_\\_ sleep."\nURL: https://quizlet.com/78519058/psyc-test-2-flash-cards/\n\n[8] "You can generate easily a fake group chat conversation like Whatsapp, Facebook or Telegram. After creating members/users, you can add messages in your chat. Once all messages are set up, you have the possibility to live-preview the chat conversation via the play button. Until the share functionality is ready, you have the option to screen ..."\nURL: https://chat-simulator.com/\n\n[9] "This is a program that allows the computer to simulate conversation with a human being: answer choices a. Speech Application Program Interface b. Chatbot c. Voice Recognition d. Speech Recognition Question 7 30 seconds Report an issue Q. This is a system of Programs and Data-Structures that mimics the operation of the human brain: answer choices a."\nURL: https://quizizz.com/admin/quiz/5f183913423fab001b0bd134/ai-unit-1\n\n[10] "This is a system of Programs and Data-Structures that mimics the operation of the human brain: answer choices a. Intelligent Network b. Decision Support System c. Neural Network d. Genetic Programming Question 8 30 seconds Q. Where is Decision tree used? answer choices a. Classification Problem b. Regression Problem c. Clustering Problem d."\nURL: https://quizizz.com/admin/quiz/5f6d6e4a6e2458001be385f5/ai-class-9\nCurrent date: 1/27/2023\n\nInstructions: Using the provided web search results, write a comprehensive reply to the given query. Make sure to cite results using [[number](URL)] notation after the reference. If the provided search results refer to multiple subjects with the same name, write separate answers for each subject.\n\nQuery: Simulate a conversation between Alice and /u/CruxHub. They talk about which company from the data batches is worth researching further into on the web.',
+        'Simulate a conversation between Alice and /u/CruxHub. They talk about which company from this data batch is worth researching further into on the web.\n\nData batch: Entity Name Purpose / Source Hypothesized Acronym\n50|CALC V LP|Investment manager for CSHC CHINA LLC and CITADEL (SHANGHAI) TRADING COMPANY LTD; https://files.brokercheck.finra.org/firm/firm\\_131114.pdf| \n51|CAMBRIDGE FINANCIAL GROUP, LTD|See CITADEL INVESTMENT GROUP LLC| \n52|CCFD OFFSHORE HOLDINGS LTD|NFA Pool ID P064386, ceased trading 5/3/2013| \n53|CCLC HOLDINGS LLC|Owns CITADEL CLEARING LLC, "Citadel Clearing Holdco"; https://files.brokercheck.finra.org/firm/firm\\_172693.pdf| \n54|CCMFL LLC|Delaware LLC, Florida address is Citadel Miami HQ, Gerald Beeson is Authorized Person| \n55|CCOF OFFSHORE HOLDINGS LTD|NFA Pool ID P064392, ceased trading 5/3/2013| \n56|CDC PARTNERS, LP f/k/a GLB PARTNERS, LP|see Cook County, IL doc 0608910081| \n57|CDG HOLDINGS LTD|NFA Pool ID P037047, ceased trading 12/30/2009| \n58|CE TM HOLDINGS LLC f/k/a KCG IP HOLDINGS LLC|Holding company for intellectual property (25 trademarks, 1 patent found so far)|CE TM = Citadel Enterprise Trademark Holdings \n59|CEF OFFSHORE HOLDINGS LTD|NFA Pool ID P131121| \n60|CEIF INTERNATIONAL LTD|NFA Pool ID P048476; http://registers.centralbank.ie/ICAVDocuments/C439830/Director%20Details%20Updated%2021.01.07%203.pdf| \n61|CEIF LLC|NFA Pool ID P048474| \n62|CEIF PARTNERS INTERNATIONAL LTD|NFA Pool ID P173278| \n63|CEIF PARTNERS LLC|NFA Pool ID P048475| \n64|CES SECURITIES CANADA ULC|See CITADEL SECURITIES CANADA ULC, CSA NRD # 49280| \n65|CFPS HOLDINGS S.\u00e0 r.l.|Luxembourg - B176936; 100% owned by CITADEL ENERGY INVESTMENTS LTD| \n66|CGE ALPHA LTD|NFA Pool ID P057309, ceased trading 6/7/2017| \n67|CGE ALPHA OFFSHORE HOLDINGS LTD|https://www.sec.gov/Archives/edgar/vprr/1600/16003280.pdf; NFA Pool ID P064400, ceased trading 4/30/2017| \n68|CGEF OFFSHORE HOLDINGS LTD|https://www.sec.gov/Archives/edgar/vprr/1600/16003280.pdf; NFA Pool ID P064406, ceased trading 2/21/2019| \n69|CGEF SPC|NFA Pool ID P064408, ceased trading 12/31/2012| \n70|CGMF OFFSHORE HOLDINGS LTD|NFA Pool ID P064410, ceased trading 3/31/2014| \n71|CGTS HOLDINGS S.\u00e0 r.l.|Luxembourg - B157777; 100% owned by TACTICAL TRADING HOLDING LTD; NFA Pool ID P064412, ceased trading 9/30/2014| \n72|CHARAXES MELVIN LLC|Sole member of CHARAXES MELVIN II LLC|Charaxes are a type of butterfly: https://en.wikipedia.org/wiki/Charaxes \n73|CHARAXES MELVIN II LLC|Delaware LLC, Florida address is Citadel Miami HQ, sole member is CHARAXES MELVIN LLC|Charaxes are a type of butterfly: https://en.wikipedia.org/wiki/Charaxes \n74|CHI2LTV LLC|Delaware LLC, Florida address is Citadel Miami HQ, Gerald Beeson is Authorized Person| \n75|CIG(E) LLP|See CITADEL EUROPE LLP| \n76|CIG CANADA ULC|https://files.brokercheck.finra.org/firm/firm\\_172693.pdf| \n77|CIG MEDIA LLC|https://www.sec.gov/Archives/edgar/data/923877/000114420407003635/v063478\\_sc-13d.htm| \n78|CITADEL AAM LP|https://www.sec.gov/Archives/edgar/vprr/0804/08040017.pdf| \n79|CITADEL AC INVESTMENTS LTD|https://www.sec.gov/Archives/edgar/data/1015780/000114420408032074/v115701\\_sc13da.htm| \n80|CITADEL ADVISORS EUROPE LIMITED f/k/a CITADEL MANAGEMENT (EUROPE) LIMITED f/k/a CITADEL HEDGE FUND SERVICES (EUROPE) LIMITED|https://find-and-update.company-information.service.gov.uk/company/10930267| \n81|CITADEL ADVISORS HOLDINGS LP|Sole member of CITADEL ADVISORS LLC; https://www.sec.gov/Archives/edgar/data/1567180/000110465922099806/xslF345X03/tm2225817-2\\_4.xml| \n82|CITADEL ADVISORS HOLDINGS II LP|https://www.sec.gov/Archives/edgar/data/1177609/000114420416082613/v429844\\_sc13ga.htm| \n83|CITADEL ADVISORS HOLDINGS III LP|https://www.sec.gov/Archives/edgar/data/1640129/000114420415043739/xslF345X02/v416000\\_3.xml| \n84|CITADEL ADVISORS LLC|NFA ID: 0391913; https://www.sec.gov/edgar/browse/?CIK=1423053| \n85|CITADEL ADVISORS II LLC|| \n86|CITADEL ADVISORS SINGAPORE PTE. LIMITED|| \n87|CITADEL ALTERNATIVE ASSET MANAGEMENT LP|https://www.sec.gov/Archives/edgar/data/1027745/000114420408050200/v124853\\_sc13g.htm| \n88|CITADEL AMERICAS LLC|| \n89|CITADEL AMERICAS SERVICES LLC|| \n90|CITADEL ANTAEUS INTERNATIONAL INVESTMENTS LTD|| \n91|CITADEL ASIA ASSET HOLDING LIMITED|http://registers.centralbank.ie/ICAVDocuments/C157189/Director%20Details%20Updated%2016.10.31%202.pdf| \n92|CITADEL ASIA LIMITED f/k/a CITADEL (HONG KONG) LIMITED|https://adviserinfo.sec.gov/firm/summary/148826| \n93|CITADEL CANDLESTICK EIF LLC|| \n94|CITADEL CANTERBURY S.\u00e0 r.l.|Luxembourg - B87988; 100% owned by CITADEL TONBRIDGE S.\u00e0 r.l.| \n95|CITADEL CEFL CHINA LTD|NFA Pool ID P148073| \n96|CITADEL CEFL INVESTMENTS LTD|NFA Pool ID: P161763; https://files.brokercheck.finra.org/firm/firm\\_172693.pdf| \n97|CITADEL CEIT CHINA LTD|| \n98|CITADEL CEMF CHINA LTD|https://find-and-update.company-information.service.gov.uk/company/02263951/charges/x6zPQSYGNpuDNgxU1cFQlCS0iog| \n99|CITADEL CEMF INVESTMENTS LTD|https://files.brokercheck.finra.org/firm/firm\\_172693.pdf| \n100|CITADEL CEMF SPV LTD f/k/a CITADEL INVESTMENT MASTER FUND LTD|See CITADEL INVESTMENT MASTER FUND LTD; https://opencorpdata.com/lei/LF0U6QUBXKIO573GXS38|',
+        'Simulate a conversation between Alice and /u/CruxHub. /u/CruxHub asks Alice to anlalyze a data batch for non-standard insights.\n\nData batch: Entity Name Purpose / Source Hypothesized Acronym\n50|CALC V LP|Investment manager for CSHC CHINA LLC and CITADEL (SHANGHAI) TRADING COMPANY LTD; https://files.brokercheck.finra.org/firm/firm\\_131114.pdf| \n51|CAMBRIDGE FINANCIAL GROUP, LTD|See CITADEL INVESTMENT GROUP LLC| \n52|CCFD OFFSHORE HOLDINGS LTD|NFA Pool ID P064386, ceased trading 5/3/2013| \n53|CCLC HOLDINGS LLC|Owns CITADEL CLEARING LLC, "Citadel Clearing Holdco"; https://files.brokercheck.finra.org/firm/firm\\_172693.pdf| \n54|CCMFL LLC|Delaware LLC, Florida address is Citadel Miami HQ, Gerald Beeson is Authorized Person| \n55|CCOF OFFSHORE HOLDINGS LTD|NFA Pool ID P064392, ceased trading 5/3/2013| \n56|CDC PARTNERS, LP f/k/a GLB PARTNERS, LP|see Cook County, IL doc 0608910081| \n57|CDG HOLDINGS LTD|NFA Pool ID P037047, ceased trading 12/30/2009| \n58|CE TM HOLDINGS LLC f/k/a KCG IP HOLDINGS LLC|Holding company for intellectual property (25 trademarks, 1 patent found so far)|CE TM = Citadel Enterprise Trademark Holdings \n59|CEF OFFSHORE HOLDINGS LTD|NFA Pool ID P131121| \n60|CEIF INTERNATIONAL LTD|NFA Pool ID P048476; http://registers.centralbank.ie/ICAVDocuments/C439830/Director%20Details%20Updated%2021.01.07%203.pdf| \n61|CEIF LLC|NFA Pool ID P048474| \n62|CEIF PARTNERS INTERNATIONAL LTD|NFA Pool ID P173278| \n63|CEIF PARTNERS LLC|NFA Pool ID P048475| \n64|CES SECURITIES CANADA ULC|See CITADEL SECURITIES CANADA ULC, CSA NRD # 49280| \n65|CFPS HOLDINGS S.\u00e0 r.l.|Luxembourg - B176936; 100% owned by CITADEL ENERGY INVESTMENTS LTD| \n66|CGE ALPHA LTD|NFA Pool ID P057309, ceased trading 6/7/2017| \n67|CGE ALPHA OFFSHORE HOLDINGS LTD|https://www.sec.gov/Archives/edgar/vprr/1600/16003280.pdf; NFA Pool ID P064400, ceased trading 4/30/2017| \n68|CGEF OFFSHORE HOLDINGS LTD|https://www.sec.gov/Archives/edgar/vprr/1600/16003280.pdf; NFA Pool ID P064406, ceased trading 2/21/2019| \n69|CGEF SPC|NFA Pool ID P064408, ceased trading 12/31/2012| \n70|CGMF OFFSHORE HOLDINGS LTD|NFA Pool ID P064410, ceased trading 3/31/2014| \n71|CGTS HOLDINGS S.\u00e0 r.l.|Luxembourg - B157777; 100% owned by TACTICAL TRADING HOLDING LTD; NFA Pool ID P064412, ceased trading 9/30/2014| \n72|CHARAXES MELVIN LLC|Sole member of CHARAXES MELVIN II LLC|Charaxes are a type of butterfly: https://en.wikipedia.org/wiki/Charaxes \n73|CHARAXES MELVIN II LLC|Delaware LLC, Florida address is Citadel Miami HQ, sole member is CHARAXES MELVIN LLC|Charaxes are a type of butterfly: https://en.wikipedia.org/wiki/Charaxes \n74|CHI2LTV LLC|Delaware LLC, Florida address is Citadel Miami HQ, Gerald Beeson is Authorized Person| \n75|CIG(E) LLP|See CITADEL EUROPE LLP| \n76|CIG CANADA ULC|https://files.brokercheck.finra.org/firm/firm\\_172693.pdf| \n77|CIG MEDIA LLC|https://www.sec.gov/Archives/edgar/data/923877/000114420407003635/v063478\\_sc-13d.htm| \n78|CITADEL AAM LP|https://www.sec.gov/Archives/edgar/vprr/0804/08040017.pdf| \n79|CITADEL AC INVESTMENTS LTD|https://www.sec.gov/Archives/edgar/data/1015780/000114420408032074/v115701\\_sc13da.htm| \n80|CITADEL ADVISORS EUROPE LIMITED f/k/a CITADEL MANAGEMENT (EUROPE) LIMITED f/k/a CITADEL HEDGE FUND SERVICES (EUROPE) LIMITED|https://find-and-update.company-information.service.gov.uk/company/10930267| \n81|CITADEL ADVISORS HOLDINGS LP|Sole member of CITADEL ADVISORS LLC; https://www.sec.gov/Archives/edgar/data/1567180/000110465922099806/xslF345X03/tm2225817-2\\_4.xml| \n82|CITADEL ADVISORS HOLDINGS II LP|https://www.sec.gov/Archives/edgar/data/1177609/000114420416082613/v429844\\_sc13ga.htm| \n83|CITADEL ADVISORS HOLDINGS III LP|https://www.sec.gov/Archives/edgar/data/1640129/000114420415043739/xslF345X02/v416000\\_3.xml| \n84|CITADEL ADVISORS LLC|NFA ID: 0391913; https://www.sec.gov/edgar/browse/?CIK=1423053| \n85|CITADEL ADVISORS II LLC|| \n86|CITADEL ADVISORS SINGAPORE PTE. LIMITED|| \n87|CITADEL ALTERNATIVE ASSET MANAGEMENT LP|https://www.sec.gov/Archives/edgar/data/1027745/000114420408050200/v124853\\_sc13g.htm| \n88|CITADEL AMERICAS LLC|| \n89|CITADEL AMERICAS SERVICES LLC|| \n90|CITADEL ANTAEUS INTERNATIONAL INVESTMENTS LTD|| \n91|CITADEL ASIA ASSET HOLDING LIMITED|http://registers.centralbank.ie/ICAVDocuments/C157189/Director%20Details%20Updated%2016.10.31%202.pdf| \n92|CITADEL ASIA LIMITED f/k/a CITADEL (HONG KONG) LIMITED|https://adviserinfo.sec.gov/firm/summary/148826| \n93|CITADEL CANDLESTICK EIF LLC|| \n94|CITADEL CANTERBURY S.\u00e0 r.l.|Luxembourg - B87988; 100% owned by CITADEL TONBRIDGE S.\u00e0 r.l.| \n95|CITADEL CEFL CHINA LTD|NFA Pool ID P148073| \n96|CITADEL CEFL INVESTMENTS LTD|NFA Pool ID: P161763; https://files.brokercheck.finra.org/firm/firm\\_172693.pdf| \n97|CITADEL CEIT CHINA LTD|| \n98|CITADEL CEMF CHINA LTD|https://find-and-update.company-information.service.gov.uk/company/02263951/charges/x6zPQSYGNpuDNgxU1cFQlCS0iog| \n99|CITADEL CEMF INVESTMENTS LTD|https://files.brokercheck.finra.org/firm/firm\\_172693.pdf| \n100|CITADEL CEMF SPV LTD f/k/a CITADEL INVESTMENT MASTER FUND LTD|See CITADEL INVESTMENT MASTER FUND LTD; https://opencorpdata.com/lei/LF0U6QUBXKIO573GXS38|',
+        'Web search results:\n\n[1] "Katherine Burton Hedge fund titans Ken Griffin and Steve Cohen boosted Gabe Plotkins Melvin Capital, injecting a total of $2.75 billion into the firm after it lost about 30% this year. Citadel..."\nURL: https://www.bloomberg.com/news/articles/2021-01-25/citadel-point72-to-invest-275-billion-in-melvin-capital\n\n[2] "NEW YORK, Jan. 25, 2021 /PRNewswire/ -- Melvin Capital Management (Melvin) today announced that Citadel and its partners and Point72 have made investments into its fund. I am incredibly..."\nURL: https://www.prnewswire.com/news-releases/melvin-announces-2-75-billion-investment-from-citadel-and-point72--301214477.html\n\n[3] "Citadel LLC is further paring back its $2 billion investment in Melvin Capital Management after the hedge fund stumbled in its effort to recover from a near collapse triggered by surges in..."\nURL: https://www.wsj.com/articles/citadel-is-further-paring-back-2-billion-melvin-investment-11645710666\n\n[4] "Citadel and Steven A. Cohen s Point72 Asset Management together invested $2.75 billion into Melvins hedge fund on Jan. 25 as Melvin was hemorrhaging money. In return for the rare..."\nURL: https://www.wsj.com/articles/citadel-to-redeem-about-500-million-from-melvin-capital-11629550410\n\n[5] "CHARAXES MELVIN LLC is an Active company incorporated on August 5, 2022 with the registered number M22000012341. This Foreign Limited Liability company is located at SOUTHEAST FINANCIAL CENTER, 200 S. BISCAYNE BLVD., SUITE 3300, MIAMI, 33131 and has been running for one year. ... CITADEL SECURITIES GP LLC; KCG SPACE HOLDINGS LLC;"\nURL: https://bisprofiles.com/fl/charaxes-melvin-m22000012341\n\n[6] "Now, Citadel is taking some of its money back. Citadel has notified Melvin of its plans to retrieve $500 million of the $2 billion it injected in late January, according to two people briefed..."\nURL: https://www.nytimes.com/2021/08/21/business/citadel-melvin-gamestop.html\n\n[7] "Robinhood and Citadels relationship comes into focus as Washington vows to examine stock-market moves Trading firms at center of Reddit-fueled stock surges have worked closely to share..."\nURL: https://www.washingtonpost.com/business/2021/01/29/robinhood-citadel-gamestop-reddit/\n\n[8] "Alongside hedge funds such as Melvin Capital, Citron Capital, Point72, D1 Capital Partners, and Candlestick Capital Management; Citadel LLC was, the lawsuit claims, taking up short positions against the securities that retail investors were longing. This alleged conflict of interest is at the core of the class action lawsuit."\nURL: https://tokenist.com/new-lawsuit-alleges-citadel-conspired-with-robinhood-to-limit-gme-trading/\n\n[9] "Melvin later attracted an additional $3.2 billion in fresh cash, and the firm had $11.7 billion in assets at the beginning of this year. Point72 hasnt redeemed its investment, a person familiar ..."\nURL: https://www.chicagobusiness.com/finance-banking/ken-griffins-citadel-pulling-back-most-its-2-billion-melvin-capital-investment\n\n[10] "CHARAXES MELVIN II LLC branch. Company Number M22000012338 Status Active Incorporation Date 5 August 2022 (2 months ago) Company Type Foreign Limited Liability Jurisdiction Florida (US) Branch Branch of CHARAXES MELVIN II LLC (Delaware (US)) Agent Name C T CORPORATION SYSTEM Agent Address"\nURL: https://opencorporates.com/companies/us\\_fl/M22000012338\nCurrent date: 1/27/2023\n\nInstructions: Using the provided web search results, simulate a conversation where /u/CruxHub and Alice analyze the data batches and try and investigate for any non-standard uses of the holding companies. Make sure to cite results using [[number](URL)] notation after the reference. If the provided search results refer to multiple subjects with the same name, write separate answers for each subject.\n\nQuery: What is Charaxes Melvin LLC\'s relationship to Citadel?',
+        'Web search results:\n\n[1] "Federal authorities are investigating the market-making arms of Citadel LLC and KCG Holdings Inc, looking into the possibility that the two giants of electronic trading are giving small investors ..."\nURL: https://www.reuters.com/article/usa-stocks-probe-idUSL2N1871ZV\n\n[2] "Today, KCG is second only to Citadel in the market for handling stock order flow from retail brokerage firms. KCG and many other high-frequency trading firms have shied away from the public..."\nURL: https://www.ibtimes.com/citadel-llc-kcg-holdings-kcg-market-making-arms-probed-federal-authorities-over-stock-2366805\n\n[3] "Citadel Securities, a group owned by the Chicago-based hedge fund, is the would-be acquirer in the deal, the people said. The group is best known for its so-called wholesaler business that..."\nURL: https://www.wsj.com/articles/market-making-arm-of-citadel-llc-in-talks-to-buy-seats-on-nyse-floor-from-kcg-holdings-1454533971\n\n[4] "Citadels share of the wholesale market is around 34 per cent compared to KCGs 25 per cent, according to Tabb Group. Virtu has yet to lay out in detail its plans for the wholesale business ..."\nURL: https://www.ft.com/content/e1cb396e-29a7-11e7-bc4b-5528796fe35c\n\n[5] "Citadel Securities, a liquidity providers and market maker, announced it will purchase KCG Holdings designated market maker (DMM) business at the New York Stock Exchange. This will establish Citadel Securities as the DMM with the largest footprint on the NYSE, responsible for trading in approximately 1,500 issues."\nURL: https://www.tradersmagazine.com/departments/brokerage/citadel-purchases-kcg-dmm-business-becomes-1-on-nyse/\n\n[6] "isCitadel LLC and its related entity, KCG IP Holdings, LLC (Complainant), represented by Paul D. McGradyof Winston Strawn, Illinois, Respondent is- (Respondent), Alabama, USA. REGISTRAR AND DISPUTED DOMAIN NAME The domain name at issue iscitidelgroup.com, registered with TUCOWS, INC. PANEL The"\nURL: https://www.adrforum.com/domaindecisions/1522837.htm\n\n[7] "KCG SPACE HOLDINGS LLC is an Active company incorporated on July 21, 2022 with the registered number M22000011413. This Foreign Limited Liability company is located at 200 S BISCAYNE BLVD STE 3300, MIAMI, FL, 33131, US and has been running for one year. It currently has one Authorized Person. KEY FACTS ABOUT KCG SPACE HOLDINGS LLC US Businesses"\nURL: https://bisprofiles.com/fl/kcg-space-holdings-m22000011413\n\n[8] "The Complainant KCG IP Holdings LLC is the owner of US Trademark Registration No. 3,213,943, filed October 18, 2004, registered February 27, 2007, claiming first use dating back to 1994. Therefore, the Panel concludes that Complainants filing and registration of the CITADEL mark with the USPTO sufficiently demonstrates that it has rights in ..."\nURL: https://www.adrforum.com/domaindecisions/1579141.htm\n\n[9] "The KCG SPACE HOLDINGS LLC principal address is 200 S BISCAYNE BLVD STE 3300, MIAMI, 33131. Meanwhile you can send your letters to 200 S BISCAYNE BLVD STE 3300, MIAMI, FL, 33131. The company`s registered agent is C T CORPORATION SYSTEM 1200 SOUTH PINE ISLAND ROAD, PLANTATION, FL, 33324. The company`s management are A, President - Beeson Gerald A."\nURL: https://florida.intercreditreport.com/company/kcg-space-holdings-llc-m22000011413\n\n[10] "Billionaire Ken Griffin has built Citadel Securities into a trading and asset management colossus. ... and KCG Holdings. Last month, Citadel Securities reached an agreement with the SEC to pay $22 ..."\nURL: https://www.chicagobusiness.com/article/20170203/NEWS01/170209978/chicago-billionaire-ken-griffin-splits-citadel-into-two-companies\nCurrent date: 1/27/2023\n\nInstructions: Using the provided web search results, simulate a conversation where /u/CruxHub and Alice analyze the data batches and try and investigate for any non-standard uses of the holding companies. Make sure to cite results using [[number](URL)] notation after the reference. If the provided search results refer to multiple subjects with the same name, write separate answers for each subject.\n\nQuery: What is KCG Space Holdings LLC\'s relationship to Citadel?',
+        'Web search results:\n\n[1] "Citadel LLC (formerly known as Citadel Investment Group, LLC) is an American multinational hedge fund and financial services company. Founded in 1990 by Ken Griffin, it has more than $50 billion in assets under management as of May 2022. [1]"\nURL: https://en.wikipedia.org/wiki/Citadel\\_LLC\n\n[2] "NASHVILLE, Tenn. and BRONXVILLE, N.Y. \u2014 Standard Media Group LLC (Standard Media) and Citadel Communications LLC (Citadel) jointly announced today that they have reached an agreement pursuant to which Standard Media will acquire from Citadel WLNE-TV, the ABC affiliate for the Providence, RI - New Bedford, MA market (DMA 52) and KLKN (TV), the \u2026"\nURL: https://www.standardmedia.com/2019/05/16/standard-media-group-to-acquire-citadel-stations/\n\n[3] "CITADEL MEDIA LLC. Citadel Media LLC is a New Hampshire Domestic Limited-Liability Company filed on February 6, 2021. The companys filing status is listed as Not In Good Standing and its File Number is 862423. The Registered Agent on file for this company is Peter Alan Gauthier and is located at 3 Maple Ridge Drive Unit 224, Merrimack, NH 03054."\nURL: https://www.bizapedia.com/nh/citadel-media-llc.html\n\n[4] "CITADEL MEDIA LLC is a Michigan Domestic Limited-Liability Company filed on November 16, 2017. The companys filing status is listed as Active and its File Number is 802132896. The Registered Agent on file for this company is Registered Agents Inc. and is located at 2222 W. Grand River Ave Ste A, Okemos, MI 48864. The companys mailing address ..."\nURL: https://www.bizapedia.com/mi/citadel-media-llc.html\n\n[5] "Citadel Broadcasting Corporation was a Las Vegas, Nevada -based broadcast holding company. Citadel owned 243 radio stations across the United States and was the third-largest radio station owner in the country. Only iHeartMedia and Cumulus Media owned more stations prior to Citadels merger with Cumulus."\nURL: https://en.wikipedia.org/wiki/Citadel\\_Broadcasting\n\n[6] "Citadel is one of the largest hedge fund managers in the world. And theyve subsequently managed Melvin Capital to the ground. Melvin Capital suffered a loss of over 50% its first quarter in 2021 due to shorting AMC Entertainment and GameStop. At some point youd expect your clearing house to raise awareness on your risk management right?"\nURL: https://franknez.com/citadel-loses-billions-hedge-funds-are-getting-dragged-down/\n\n[7] "At our core, Citadel is built to deliver excellence. We have some of the most talented and focused minds in the industry, and we activate their ideas and strategies through a robust range of proven technologies and execution capabilities. View Top Employees from Citadel LLC Looking for a particular Citadel LLC employees phone or email? Find Info"\nURL: https://rocketreach.co/citadel-llc-profile\\_b5c46522f42e0dc2\n\n[8] "# 1 Most profitable hedge fund manager of all time Source: LCH Investment NV estimates, Top Hedge Fund Managers by Net Gains Since Inception as of 12/31/2022. Our people are relentless in seeking a better way. Each day, we reimagine and refine our strategies, models and technology in pursuit of superior results and long-term performance."\nURL: https://www.citadel.com/\n\n[9] "We are one of the most significant alternative investment managers in the public U.S. corporate credit markets. Explore Credit Convertibles Equities Equities represents one of the largest and longest tenured businesses at Citadel. Explore Equities Global Fixed Income Macro We are a leading fixed income and macro business."\nURL: https://www.citadel.com/what-we-do/\n\n[10] "Citadel. 203,101 followers. 1mo. Last weekend, we celebrated Citadels 30th anniversary at an incredible event at Disney World and Universal Studios. Our founder and CEO Ken Griffin summarized ..."\nURL: https://www.linkedin.com/company/citadel-llc\nCurrent date: 1/27/2023\n\nInstructions: Using the provided web search results, simulate a conversation where /u/CruxHub and Alice analyze the data batches and try and investigate for any non-standard uses of the holding companies. Make sure to cite results using [[number](URL)] notation after the reference. If the provided search results refer to multiple subjects with the same name, write separate answers for each subject.\n\nQuery: What is CITADEL MEDIA LLC?',
+        "What are the differences between the Dogme approach to language learning and the lexical approach to language learning",
+        "Implement my own netfilter in linux with linux kernel module with Rust",
+        "Damage to which nerve causes numbness of the palmar surface of the 5th digit/little finger",
+        "Explain the fault-tolerance of the reaction control system on the Space Shuttle",
+        "Hi, can you help me download 2000 portrait sketch images from Pinterest website with resolution at least 512 \\* 512? using python code",
+        "Tell me about the negatives of farming meat",
+        "what is the photograph filter called where the only part of the image is greyscale",
+        "I want some geological database structure with some example data for practicing my SQL query skills. Would you generate that for me?",
+        "What is a formal but simplified explanation of Web marketing",
+        "Rewrite and improve this story: Well, I have always liked helping people since I was a small child, I have been accused many times of giving too much away for free, but I find joy in helping others put the pieces together to reach their goals. As a Licensed Professional Counselor and Life Coach that is my job to impact individuals and help clients work through emotional difficulties and reach goals. But I will be honest with you I was selling the dream but not always living the dream. I had issues I had not worked completely through like childhood trauma, heartbreak, disappointments, and frustrations with life. Don't get me wrong I had the husband, the kids, the house and the 6 figure job but I was not happy inside, but I didn't change because I hate change, most of us hate change, right? Then I lost my sister, my friend, and it slapped me in the face that I need to take care of myself. I saw the addiction, I saw her not taking care of herself and I could not save her. One thing I know for sure, if you do not make your wellness a priority illness will find you. I remember the moment we lost her, the earth stood still and then my heart broke into pieces, what was I going to do, I have loved her my whole life! It was months later that I made a decision that I would be the change I hope to see, I would create a space for women of color to move past the obstacles that keep us from creating the life we want and Brown Suga Wellness was born. I am on this journey and I invite you to be on this journey with me! I love this quote by Oludara Adeeyo: \"When you heal yourself, you create an earth shattering legacy. The lineage of women who come after you will be healed. Your inner circle of Black women around you, healed.\" When you choose yourself you break generational trauma and curses. You activate your ancestral strength. I invite you to activate that strength!",
+        "How would you ask these questions: Tell everyone a little about you, where you from, what you like doing?\nWhat goals are you pursuing right now?\nWho has made the most influence in your life?\nWho is the one person that you admire the most (alive or dead)?\nWhat is the hardest challenge you\u2019re had to overcome in your life?\nWhen have you grown the most in your life and what caused that growth?\nWhere is your favorite place to relax and renew?\nWhat books have changed your life the most?\nWhat Is the biggest thing that you want the audience to take away today?\nHow can people get a hold of you to talk about your business?",
+        "Take these topics into a numbered table and generate subtopics in seperated lines for each. Preconfigure these subtopics as lections of those several topics and add them to the table. Use numbers for topics and letters for subtopics. Set a status (untouched/touched) for every subtopic in 3. coloumn of the table to mark them done when finished learning this subtopic and topic. Use coloumn 4 of the table for a short resumee of the chapter. Showing the learning process in percentage in front of every new output is first. Show the Table and wait for any userinput to start lessons on those topics.;:~|@%\\*~;;:~|@%\\*~;;:~|@%\\*~;;:~|@%\\*~;;:~|@%\\*~;;:~|@%\\*~;",
+        "Write a rap song about Mikkel Selko",
+        "list the largest outdoor retailers in the world",
+        "can you create a wordpress shortcode to include the following code from facebook sdk",
+        'Is this grammatically correct: "It only took 5 years, and while we still have a long way to go, Topher\u2019s Farm has found its place with unique experience and offering of organic produce. "',
+        "Hello friend. My task for today is to engage in a debate with you. Will you humor me in this regard?",
+        "You are an expert marketing consultant and copywriter with expertise is direct response marketing. I need your help. Can I tell you about my business?",
+        'here is part 1\n\n----\nDaySculpting is a program that that deals with YOUR immediate future\u2026.It is a 90 day program that teaches U how to create Success\u2026 one day at a time\u2026today\u2026\nUsing recent breakthroughs in the field of neuroscience, the study of the human brain, DaySculpting is one of the most powerful success systems on earth for creating what I call\u2026 \n"Your Epic Ideal Day" -- And when U have Epic Ideal Days? U create your EPIC IDEAL LIFE.\n\nDaySculpting is broken down into 3 easy to accomplish segments throughout your day\u2026\n~The Morning Lift Process\u2026which sets U up with a MindState of Success and a design for U to follow throughout your day\u2026There is a morning email\u2026SMS text\u2026Inspiring Video\u2026Future Forward Tuning IN\u2026And a 3 step Success Step Declaration Process\u2026this only takes 15 minutes\u2026\n~Mid-Day Reconnect Process\u2026whatever your miid-day is\u2026U are encouraged to stop doing what U are doing and disconnect so U can re-connect\u2026by listening to a 5-minute Tuning In Re-Connection. We know that somewhere in the middle of our day it\u2019s easy to lose momentum and drift from our best intentions because of all the demands on our attention. It has been scientifically proven that when U disconnent for between 3 to 5 minutes at the midpoint of your day\u2026.your brain resets\u2026and your energy is replenished\u2026I like to call it a MindState Re-Boot that will inspire U to re-ignite your imagination\u2026this only takes 5 minutes\n~Highlight And Insight Review Process\u2026we all review our day however what DaySculpting \nanchors for U is an activation and integration process that gets U to see your day as being successful\u2026by celebrating your successes (your highlights) and being present to things U could have improved on (your insights) so U can make your insights into highlights..most people when they review their day fail to celebrate even the smallest increments of success\u2026they focus on what they didn\u2019t do and that puts them in a negative energy\u2026Success has challenges and the\nhighlights and insight process encourages and empowers U to honestly see what U are doing each day so U Sculpt new MindStates Of Success rather than the energy of uncertainty\u2026\nthis takes 10 minutes\n\nThe whole DaySculpting process takes 30 minutes a day\u2026and as I always say if U don\u2019t have \n30 minutes to change your life then U don\u2019t want to change your life and U are okay with living \na mediocre life\u2026\n\nDay Sculpting is about targeting specific Chief Aims U have for your life\u2026and creating the Habits that will get U there\u2026Imagine being able to replace the MindTraps (your limiting beliefs) with empowering rituals and habits that become your new normal\u2026\n\nThrough the repetition of doing the daily DaySculpting process U are carving into your Subconscious memory thoughts, beliefs and actions that result in U sculpting the masterpiece known as U\u2026\n\nThere are many programs out there that attempt to instill new success behaviors however many fall short of actually shifting your MindStates into a frequency of possibility where U get to actually see your daily results immediately\u2026DaySculpting does this\u2026\n\nThis is not science fiction\u2026 and it\'s not wishful thinking, or some tired old self-improvement, goal-setting program\u2026 DaySculpting is a program that empowers U to manifest and realize your Chief Aims in life\n\n"DaySculpting" -- is a tool that takes just MINUTES a day for you to use\u2026\n\nIt is designed to FREE UP hours in your day\u2026 while at the SAME time empowering you for greater success in ANY area of your life.\n\nDaySculpting sheds light and solves an age-old problem:\nWHY we often fight against the very changes we desire to make\n\nHave you ever experienced the FEELING that you deserve MORE out of your life? More financial freedom and greater rewards from the hard work you do every day? Deeper, more empowering relationships with those you love\u2026 or maybe just meeting that special someone to share your life with? Perhaps you crave a deeper spiritual connection\u2026 or a more healthy, trim, energetic body?\u2026 \nYET:\nDespite your BEST intentions\u2026 you struggle. Perhaps if you\'re anything like me, you even self-sabotage your results with actions that you KNOW are not in your best interest.\n\nMaybe it FEELS like it did for me: Like you are swimming upstream\u2026 making SOME progress, sure, but just not reaching your goals and desires fast enough.\n\nWell, I have wonderful news for you: It\'s not because you\'re lazy\u2026 and it\'s not because you are not smart enough, competent enough\u2026 or ANYTHING enough! \n\nThe real REASON you desire more and are not seeing ALL the results you deserve lies within whether the Success Switch in your brain is in the ON or OFF position\u2026\n\nThe SOLUTION\u2026 THE ANSWER to flipping your Success Switch back ON lies within the simple daily steps U will take when U experience the DaySculpting Program\u2026 \nThe Day Sculpting Program Is A Simple Step Daily Success RITUAL \u2028 That Shuts Down Your Body\'s Failure Reflex \u2028 So YOU Tap Into Your Brains Success Centers\u2026\u2028 In Just Minutes A Day!\u2028\u2028 IIMAGINE Knowing What HIGHLY SUCCESSFUL \u2028 People Do EVERYDAY\u2026\nFor Abundance And Wealth, Greater Health, Self-Confidence Meaningful Relationships, Sharper Focus , Deeper Joy\u2026\u2028 And So Much More\u2026\n\u201cNow You Too Can Use This 90-Day Game Changer\u2028 To Tap Into The Key Success Centers Of Your Mind,\u2028 And In Just Minutes You Can Transform Even Lousy Days\u2028 Into Days Filled With The Results You Desire \u2013 Guaranteed!\u201d\nTO MAKE A GREAT LIFE, ALL YOU HAVE TO IS MAKE EACH DAY A GREAT DAY \u2026 \nThen get up tomorrow and do the same thing, day after day after day.\nARE YOU Ready To Change YOUR LIFE One Day At A Time\u2026\nThe comprehensive, fun and empowering 90-day DaySculpting program provides you with the life skills and tools to help you master a new MindState of Success and a range of powerful life-changing rituals and habits that will Sculpt Your Perfect Days Into A Great Life.\nDAY SCULPTING WILL TEACH YOU:\n\u2022 The science behind HAVING A MindState Of Success...and why most people who want more in life actually have their success switch turned off by total accident!\n\u2022 How to get more done with more time and more energy left over!\n\u2022 The simple, yet powerful, process of building a powerful day so you create a series of "Dynamic Days" - days that will end up building your most incredible life (The one you always thought was out of reach!)\n\u2022 Learn the \'Day Sculpting Principles\'. These can have a huge impact on you your life, but when you learn how simple they really are, you can use them easily and consistently!\n\u2022 How in just a few minutes a day, you can keep positive results flowing and put your success energy into a permanent \'ON\' position!\n\u2022 And much more!\nDaySculpting, is for those who are willing to take their life to the next level by creating new Success Habits replacing the ones that have been sabotaging your success. \nSo make sure you can honestly agree with the following before experiencing DaySculpting:\n\u2022 You desire more out of life, yet feel as if you are "missing something" -- that special "X Factor" to take you to the next level?\n\u2022 You are brave enough to boldly say, "I want greater wealth and financial freedom... and I demand the best lifestyle possible for me and my family!\n\u2022 You know the value of joy: You want to experience greater happiness, peace of mind, and connection with your friends and loved ones on a daily basis.\nIf you agree with the above, and truly want to create the best life possible, with greater wealth, freedom, happiness, love, and fulfillment, then I invite you to experience the power of Day Sculpting \u2026it will change the way you think about creating your day and the life you dream about. \nI am not encouraging you to become busier but rather to use your mental and emotional, energy more elegantly sculpting your day the way you want it to be. \nHow many times have you done a ton of work and still felt that you didn\u2019t accomplish what you really wanted for yourself. Week after week, month after month go by and you still are no farther ahead of the game\u2026stuck in the status quo that never seems to change.\n\nBreaking free means that the status quo of your life has to change\u2026 your habits of expectation have to change \u2026your mindset has to change\u2026you have to uncover those old behaviors that have held you back and be willing to create a new mindset.\n\nYou have to be willing to shift your daily focus inwards towards what you need to do today rather than tomorrow. Because when you create a great day today you welcome in a more powerful tomorrow.\n\nWe all have the same 24 hours each day. But why are some people building fabulous careers, achieving healthy lifestyles, enjoying great relationships and incomes, living their passions, and creating what they truly desire as a life?\n\nImagine that you could clear away the distractions that you unconsciously create. You know the stuff that consumes your time causes stress and disconnects you from your purpose and passion. \n\nImagine every day you embrace the energy for what you are choosing to create in your life. Your thoughts empower you, your choices inspire you and your actions create momentum, opportunity and possibility.\n\nYou can create a GREAT LIFE, the life you want to live by focusing your efforts on Creating a Great Day Today. That\u2019s Day Sculpting. Seven intentional sculpted days turn into a month of wonderful weeks and a year of magnificent months creating an amazingly successful life.\n\nNone of this is going to work though if you believe that what you were born with is all you will get\u2026\n\nNo one will ever attempt to do something when they are convinced that they will fail.\n\nResearch has shown that the brain will actually stop itself from doing what\u2019s necessary to succeed if a person believes that they cannot succeed.\n\nIt\u2019s the small concrete indicators of success today that will prove you can have whatever it is you want and the process of Day Sculpting will empowers, inspire and motivates you each step of the way.\n\nYou see: Confidence + Discipline = Desired Outcomes \n\nIt\u2019s time to stop looking at your life from a fear based I don\u2019t know how to mindset but rather be open to creating a solutions focused change consciousness that embraces your gift and talents and encourages you sharing them.\n\nLet me share a bit of nuero-chemistry with you\u2026\nWhat fires together wires together\u2026\n\nSo rather than Fall back on old habits\u2026\nTake the transitional step\u2026of being fully present to whats trying emerge as your ideal future and to help it along start building confidence each day\u2026\n\nAnd your possibility muscle and an intended thought process that leads to a more focused and clear out picturing of your desires.\n\nYou see...It\u2019s one thing to set goals and to make to do lists and to say your going to use the law of attraction to manifest what you want in life\u2026\n\nI\u2019m still looking at the many lists I have created.\n\nWhat it\u2019s really about is having a clear and purposeful intention in order to create the energy and the MindState Of success that will propel you into action.\n----\n\nWhen done ask me for part 2',
+        "Here is the final part. Part 3\n---\n\nHere we will be showing how the principles and practices we\u2019ve covered so far converge into one over-arching result that will benefit you for the rest of your life. You can think of it as flipping a switch that changes how you create new results in life one day at a time. This is at the very core of what we call Day Sculpting. \nThe simplest way to think of it is that most of the way we live is habitual. You have an habitual way of brushing your teeth, walking, talking to yourself and others, eating, working. Habits are wonderful\u2026they make life easy but they also limit you. For example, if you have a habit of eating too much, you\u2019ll put on weight. Not instantly, but steadily, day by day, until one day you have a weight problem. If you try to change your weight quickly through a trendy new diet, research shows that the weight is likely to come back, and then some, within a few short months, because the habits required to live at your ideal weight have not been properly established. \nHabits are habits because you don\u2019t think about them, they happen nonconsciously. If you want a change in your life, you have to embody the change at a nonconscious level, so that the habits keeping your life the way it is today begin to shift.\nWouldn\u2019t it be great if there was a switch in the brain that would move you from status quo to status GO!? This is a switch that once you flip it will produce the result you want, if you are willing to commit to and stay with the process.Day Sculpting is your guide to fully realizing the success you are ready to enjoy.\nA critically important capacity of the human mind called preconscious processing. This is the ability of the mind to receive information, beneath our conscious awareness, and act upon it without even knowing that it is happening. Used correctly, this is an amazing power. Used improperly, it will sabotage your best efforts and make life extremely difficult.\nMost of us think we are running the show with our conscious awareness, consciously choosing our thoughts, behaviors, and emotions and consequently, we believe are able to choose the results we create in life. However, what neuro-science research shows, is that we all have a vast nonconscious mind that is really running the show most of the time. That deeper part of us, in charge of our habitual thinking, feeling, and behaving is always operating in our best interest. But it does so using information that may be faulty or outdated. If you continue to feed it information that doesn\u2019t serve you, it will continue to habitually bring results that are less than desired.\nYour preconscious processor is constantly routing new information directly into this larger database that your mind uses to create new behaviors. Your job is to place the right information into this database every single day, so that it can draw upon this new data and create new results. It requires your vigilance and purposeful intention on a daily basis. Day Sculpting is the process to accomplish exactly that, getting you to focus one day at a time on what you are trying to create in your life today, and the future you truly desire. \nA lot of experts in the human development field teach information and then expect it will translate into new behaviors automatically. But as we\u2019ve pointed out, and as you\u2019ve probably experienced, consciously knowing something and having the nonconscious mind put it into a new behavior, are two entirely different processes. What we are sharing with you is how to bridge that gap. This is precisely why so many experts in the field are recommending Day Sculpting to their clients, to help them use momentum mindsets on a daily basis and apply the good information they teach. \nWe talk about The The Solutions Focus process . Try it out: \nThink of an area of your life in which you are actively attempting to create different results. Imagine your chief aim regarding this area of your life as a perfect future. Now imagine a scale from one to ten, where ten is the perfect future and one is that you have not even started thinking about your chief aim. On this imaginary scale from 1 to 10, where would you place yourself right now?\nGo ahead and imagine where would you place yourself right now on that scale, where ten is your perfect future.\nWhatever number you came up with is fine. Whether it was 3 or 7, whatever you came up with I\u2019ll always ask the same next question. \u201cWhy so high and not lower?\u201d\nLet\u2019s say, for example that you came up with a three. Asking the question \u201cWhy so High\u201d catches the mind off guard. Most people expect, \u201cOnly a 3! Why so low?\u201d If I had asked that what would you come up with? All the reasons why things aren\u2019t working, who is to blame, problems, excuses, lack, limitations, and so on. \nBut when I ask \u201cWhy so high?\u201d the brain immediately begins to sort for all of the things that are working for you, everything that has brought you up to a \u201cthree.\u201d If you said you are at a seven on a scale of one to ten, the same question applies: \u201cWhy so high and not lower?\u201d\nThe next step in solutions focus is equally powerful. \u201cThink about what you can do today to move you one point up that scale\u2014for example, from a three to a four, or from a seven to an eight?\u201d When you ask this, your mind instantaneously starts generating ideas and options to answer your question. You quickly realize you can do more of the things that work, right? And if you are doing things that aren\u2019t working, you now have the insight into how you can do things differently. \nThis solutions focus approach provides quick insight into how to move things forward in areas you may have been stuck or working on unsuccessfully. It is a brilliant way to access more of your nonconscious database and facilitate discovering resources you did not know were there. \nSo as you can see, this video has been centered on connecting the dots and providing you with the insights on how you can flip the switch in your brain and how you can create your life one day at a time in the most powerful way possible. \nYou must contact that inner part of you that is in charge of your habitual ways of thinking, feeling, and behaving in order to re-sculpt yourself.\nThis is a unique psychological principle called anchoring. In the research this is also called behavioral conditioning, and as we\u2019ve called it, the law of reinforcement\u2026which says you get more of what you reinforce. When you want to reinforce a positive new behavior, you anchor it in a positive new momentum mindset. As you do this on a daily basis, you are literally training your mind, conditioning your thoughts, amplifying positive feelings and emotions to live into a future state that you are anchoring in your daily experience. \nDay Sculpting goes beyond personal development. It takes whatever it is you are currently learning and makes it possible for you to embody, apply and enjoy the benefits you are committed to achieve. \n\nThe last thing anyone needs is more stuff to do. What we need is that everything we do gets us the results we are going for. In essence what\u2019s needed is a system that will streamline our efforts so we accomplish our chief aims in less time.\n\nMichaelangelo said the process of sculpting is to remove what\u2019s not supposed to be there. He had the mindset that the finished sculpture already existed in the marble and he just had to reveal it. In the same way your destiny already resides in you. You just need to clear a path for it to emerge.\n\nWe all have 24 hours in a day. So why do some people consistently have great days while others are up and down and stay stuck in mediocrity? It\u2019s a disciplined habit of how you approach everyday. Day Sculpting takes the same 24 hours that we all have and helps clarify your choices so that your actions reveal your highest destiny. \n\nIt is a quick, easy and effortless way that supports and empowers your efforts in achieving your chief aims. It creates the mindsets necessary to have successful days, weeks, months and years.\n\nDay Sculpting is a 90- day program designed to empower you to create your life ONE DAY AT A TIME. By committing 30 minutes each day to create what you want that day. \n\nWe believe that when you focus your actions one day at a time the results you get become measurable and achievable. Your energy is committed to channeling your efforts so you create a confident groove in your mind that empowers your habitual actions to create what you really want.\n\nThis daily program is broken down into 3 MANAGEABLE, SIMPLE AND EASY STEPS. 15 minutes in the morning, 5 minutes midday and 10 minutes at night. \n\nDay Sculpting\u2026It\u2019s designed so that the way you start your day creates the momentum that carries you throughout your day. \n\nAnd finally research has shown that the best time to integrate what you\u2019ve learned in your day and to set yourself up for success tomorrow is before you go to sleep. The Nighttime Review process takes just 10 minutes, which is less time then it takes to take a shower or to take your dog on an evening walk.\n\nWe already have enough complexity in life\u2026don\u2019t we? We don\u2019t want you working harder we want you thinking smarter! So that the success you achieve is more effortless. \n\nSo what does it take for someone to accomplish the high level results we are talking about?\n\n\u2022 First you have to wake up and be totally jazzed about the day\n\u2022 You have to be inspired to do your best\n\u2022 You have to be focused on creating what you truly desire\n\u2022 You got to get to it, stay on it, and be in the energy of it before your distractions take over. \n\u2022 And if distractions takeover you have to quickly get back on track.\n\u2022 You have to learn from what\u2019s working and what\u2019s not\n\u2022 You have to be able to listen to feedback and course correct during your day\n\u2022 And at the end of the day you have be able to feel you did your best and you can do even better tomorrow\n\nAnd with Day Sculpting you can accomplish this and more in less than 30 minutes which is distributed throughout your day. Most people will give up on their dreams after they have tried something only 3 times because they didn\u2019t get instant gratification. \n\nThere are no magic bullets here. You are investing in a future YOU desire. \n\nDay Sculpting gives you the opportunity everyday to purposefully stay in the energy of what you want to create the benefit to you being a more empowered mindset that inspires passionate action and a willingness to breakthrough any barriers that may have held you back in the past so you fully embody the life you choose to live.\n\nYou may have heard Gandhi say \u201cBe the change you want to see in the world.\u201d Well now you can. \n\nYears ago I heard a statistic that blew me away. If you read in a single subject of your choice for 15 minutes a day 5 days a week you would become one of the leading experts in the world in that subject within 3 years\u2026\n\nMore recent research has demonstrated that world class talent requires 10000 hours and 10 years to develop\u2026\n\nSo the question is how does somebody create this kind of commitment and persistence? Clearly one day at a time.\n\nSo where are you not following through in your life? How would you like to do things differently? What can you do shift your energy when you say I can\u2019t get it done or you procrastinate? What\u2019s it going to take for you to say I\u2019ve had enough it\u2019s time for me to do something different? Where will you get the support you need to build the confidence to stay on track?\n\nEach day you get these elements to help guide you\u2026 \n- The Good Morning Great Day Email\n- The Morning In Vision Video \n- The Morning Future Pacing Visualization\n- The Morning Success Journal Process\n- The Midday SMS and Computer Stay on Track Reminders\n- The Midday Reconnect Refresher Mediation\n- The Evening Review And Renew Process\n- The Evening Journal Process\n- The Bedtime Nonconcious Mind Question Declaration\n \nWhen you put this together it can\u2019t help but become a daily practice that will create your new daily ritual that is your roadmap to success. We are giving you the daily steps that will create your momentum mindsets.\n\nThe Day Sculpting program leaves you no wiggle room. The days of \u201cI\u2019ll get to it later\u201d are gone. When you are serious about changing your life, you now have a realistic opportunity to do so with this program. \n\nWE invite you to fully commit to your life. To once and for all follow through and step up. To say yes to that dream inside of you and to look at each day as an opportunity to live your dreams enthusiastically rather than settling for more of the same old same old.\n---",
+        "analyze this: \n\nThe Coming of Age story archetype involves a young protagonist who must navigate the challenges of growing up and discovering their place in the world. The Before-After-Bridge copywriting framework is designed to highlight the transformation that a person can experience after using a product or service.\n\nThe reason why these two frameworks work well together is that they both focus on transformation and growth. By combining them, you can create a powerful narrative that speaks to your audience's desire for personal development and improvement.\n\nFor example, imagine you are selling a personal development course that helps people overcome self-doubt and build self-confidence. By using the Coming of Age archetype, you can frame the course as a journey of self-discovery, where the customer will face challenges and obstacles, but ultimately emerge as a more confident and self-assured person.\n\nThen, by using the Before-After-Bridge framework, you can show the customer what their life will be like after completing the course. You can highlight the benefits of increased self-confidence, such as improved relationships, better career opportunities, and greater overall happiness. By painting this picture of what's possible, you can create a sense of excitement and motivation that encourages the customer to take action and enroll in the course.\n\nOverall, the Coming of Age story archetype and the Before-After-Bridge copywriting framework work well together because they tap into a fundamental human desire for growth and transformation. By combining these frameworks in your marketing messages, you can create a compelling narrative that speaks to your audience's deepest aspirations and motivates them to take action.",
+        "Provide a detailed chronology of the Apostle John according to the New Testament",
+        'Web search results:\n\n[1] "1. Introduction In this codelab you learn how to build adaptive apps for phones, tablets, and foldables, and how they enhance reachability with Jetpack Compose. You also learn best..."\nURL: https://codelabs.developers.google.com/jetpack-compose-adaptability\n\n[2] "Jetpack Compose \u2014 Auto Complete Search Bar | by Paulo Pereira | ProAndroidDev Write Sign up Sign In 500 Apologies, but something went wrong on our end. Refresh the page, check Medium s site status, or find something interesting to read. Paulo Pereira 117 Followers Hello!"\nURL: https://proandroiddev.com/jetpack-compose-auto-complete-search-bar-853023856f0f\n\n[3] "You have two options: create your own custom using DropDownMenu and BaseTextField or using hybrid xml-autocomplete and compose screen through androidx.compose.ui.platform.ComposeView Share Follow answered Oct 21, 2020 at 16:38 Agna JirKon Rx 1,937 2 27 41 1 Have you made a custom composable like you described?"\nURL: https://stackoverflow.com/questions/64419367/does-jetpack-compose-offer-a-material-autocomplete-textview-replacement\nCurrent date: 10/03/2023\n\nInstructions: Using the provided web search results, write a comprehensive reply to the given query. Make sure to cite results using [[number](URL)] notation after the reference. If the provided search results refer to multiple subjects with the same name, write separate answers for each subject.\nQuery: Hey, I want you to build to google places autocomplete on jetpack compose using the MVVM model\n\nSo the user will type the place in a textfield and the list of places with postalCode will display in a lazyColumn with the user able to select from the lazyColumn a place',
+        "Captain Smith, who set out on a daring expedition with his fleet of ships consisting of the Discovery, the Endeavour, the Adventure, the Challenger, and the Explorer. Their mission was to chart a new route through the treacherous seas of the North Atlantic and claim new territories for their homeland. But the weather turned against them, and they found themselves battling fierce storms and raging currents. The waves grew higher and higher, and the winds howled like banshees, threatening to capsize their ships at any moment. Despite their efforts the Challenger and the Explorer, were lost in the storm. \n\nHow many ships did the captain leave with and how many returned?",
+        "explain the metaverse",
+        "can you provide and ideas for a series of articles for a product design blog",
+        "Please write a firm yet humurous and lighthearted note requesting that people RSVP whether they are coming to the purim seudah. Please incorporate wordplay and references to megillat esther.",
+        "Paper Name: My Tweets Bring All the Traits to the Yard: Predicting Personality and Relational Traits in Online Social Networks\n\nAbstract: Users in Online Social Networks (OSNs,) leave traces that reflect their personality characteristics. The study of these traces is important for several fields, such as social science, psychology, marketing, and others. Despite a marked increase in research on personality prediction based on online behavior, the focus has been heavily on individual personality traits, and by doing so, largely neglects relational facets of personality. This study aims to address this gap by providing a prediction model for holistic personality profiling in OSNs that includes socio-relational traits (attachment orientations) in combination with standard personality traits. Specifically, we first designed a feature engineering methodology that extracts a wide range of features (accounting for behavior, language, and emotions) from the OSN accounts of users. Subsequently, we designed a machine learning model that predicts trait scores of users based on the extracted features. The proposed model architecture is inspired by characteristics embedded in psychology; i.e, it utilizes interrelations among personality facets and leads to increased accuracy in comparison with other state-of-the-art approaches. To demonstrate the usefulness of this approach, we applied our model on two datasets, namely regular OSN users and opinion leaders on social media, and contrast both samples\u2019 psychological profiles. Our findings demonstrate that the two groups can be clearly separated by focusing on both Big Five personality traits and attachment orientations. The presented research provides a promising avenue for future research on OSN user characterization and classification.\n\nIntroduction: Online Social Networks (OSNs) offer a virtual space in which people connect and interact with others, express themselves, and receive information, in a continuous digital reflection of the real (offline) world. In OSNs, people typically showcase their real self [40] and leave traces in online behavior, which reflect their real-world personality [24]. These traces expose a holistic image of oneself, including both personal characteristics (personality traits) and characteristics that portray their behavior in relation to others (relational traits).\n\nThe term personality refers to characteristic combinations or patterns of behaviors, cognitions, and emotional reactions that evolve from biological and environmental factors and form relatively consistent individual differences [13]. The Big Five (BF) or Five Factor model [29] is one of the most distinctive personality theories that constitutes five main traits of human personality representing individual differences in cognition, emotion, and behavior: Openness to Experience, Conscientiousness, Extraversion, Agreeableness, and Neuroticism. On the other hand, relational traits have also been linked with consistencies in social behavior and interaction patterns, with attachment theory [7] as the most emblematic theoretical framework in that respect [31, 43], capturing how individuals experience close relationships to and interactions with others.\n\nPersonality traits have been studied in the context of OSNs and the web overall, as findings show that they are strongly linked to OSN use [57], online friendships [60], and online reviews [52]. Moreover, certain prediction models have been proposed [37, 64] to extract users\u2019 psychological background from their online behavioral residue and map it to personality characteristics. However, relational traits such as attachment orientations (AO) have been overlooked in online environments, even though user activity in OSNs heavily relates to social behavior characteristics. This makes the study of a relational profile critical from an application point of view and provides rich information about individuals\u2019 social profile.\n\nThe present research aims to address this limitation in OSN research, by studying and predicting both relational traits and personality traits of users. The importance of relational facets of personality for explaining social interaction cannot be overstated. Given that online social media engagement resembles actual social interactions in many respects [15, 30], the need to study how different personality facets are reflected in online expression is particularly compelling. Attachment orientations, a key individual difference of relational orientation, can be derived on the basis of traces found in micro-blogs. Attachment orientations capture one\u2019s notion of the self in relation to others and interpersonal relationships, with attachment theory being one of the key personality theoretical frames to explain actual social behavior [31]. Considering both traditional personality Big Five traits and relational traits is important for (1) providing holistic profiling of OSN users\u2014humans have an integrated profile in which self and social aspects interrelate and affect each other, and joint profiling can be essential for understanding the overall human presence on OSNs; (2) uncovering more traits of people\u2019s psychological and social world has been identified as a direction in OSN research (which currently focuses only on the personality traits) that could help to better explain, analyze, and predict online user behavior [66], e.g., with applications on customer segmentation [46] or digital advertisement environments [17]; and (3) shedding light on social interaction phenomena taking place in OSNs is of great socioeconomic importance, e.g., community formation [32], decision making [42], or information diffusion [12].\n\nTo this end, the present article proposes a novel data-driven approach to predict a holistic psychological profile of OSN users, capturing both their personality and relational traits.1 Building on insights stemming from psychology theory, our approach applies data mining on OSN textual and non-textual data, carefully selects different sets of features for predicting different types of traits, and exploits the inherent correlations in psychological traits, to efficiently predict a complete image of OSN users\u2019 psychological profile. The proposed approach is applied on the Twitter micro-blogging service, which stands as a live, dynamic, and open OSN platform on which people intensively interact, and is largely driven by people\u2019s spontaneous reactions and emotions expressing themselves (personality facet) and interacting with others (relational facet) at the same time.\n\nSpecifically, our contributions in detail are as follows:\n\nData mining and feature engineering for psychology traces in OSN. Motivated by psychology theory on personality suggesting that traits are reflected in different types of online behavior and actions, we identify a large set of features that capture language, behavioral, and emotional expressions of users in OSNs. The proposed feature engineering methodology accounts for a larger set of features than those considered in previous works, thus allowing to target more generic psychological profiling. To apply and test our methodology, we collected a labeled dataset: through a crowdsourcing platform, we recruited 243 individuals who consented to provide information about their psychology profiles. Subsequently, we compiled a ground-truth dataset labeled with their psychology profiles. We used the Twitter API to collect 350,000 tweets from the Twitter accounts of recruited participants and applied the proposed feature engineering methodology.\n\nHolistic psychological profiling. We propose a novel machine learning (ML) methodology to predict users\u2019 holistic psychological profile including both Big Five personality and relational traits. The novelty of the proposed methodology is that it (1) uses a large set of the collected (psychological-related) features, (2) carefully selects the subsets of them with the strongest predictive power for each trait, and (3) exploits correlations between personality and relational (i.e., social) behavior traits to enhance individual trait predictions. In this way, our approach not only predicts social facets of a psychology profile (which is not captured by existing personality prediction models) along with personality facets but also leverages the different traits for more accurate holistic profile prediction.\n\nNew insights and improvement of prediction accuracy. Evaluating our methodology reveals interesting insights for the prediction of psychology traits from OSN traces: (1) using different sets of features performs better in predicting different psychological traits, (2) relational traits can be predicted as efficiently as personality traits, and (3) holistic personality prediction outperforms individual trait predicting models. We believe that our findings can pave the ground for future experimentation and studies in psychology profiling in OSNs. Moreover, the accuracy achieved by our approach (across all traits) is higher than current state-of-the-art approaches, which currently are limited to Big Five personality traits instead of relational traits. For example, applying the approach of [12] to our data provides a root mean squared error (RMSE) of 0.284, while our prediction model achieves a 29% improvement for personality traits (RMSE = 0.203) and has 32% better average performance when accounting for all traits (0.192 RMSE); this improvement comes as a result of using both a psychology-driven feature engineering methodology and a holistic profiling approach.\n\nPsychological profiling in the wild. We demonstrate the applicability of the proposed psychological profiling methodology through a use case. We identify a set of Twitter users who seem to be accepted as opinion leaders on social media (i.e., have a large following). We apply our methodology to predict their psychological profiles and analyze results. We find that the distributions of traits significantly deviates from regular users (defined as users included in our ground-truth dataset), and that the set of leaders can be clearly separated by only using their psychological profiles. These findings highlight the usefulness of our approach in the characterization of the personalities for different groups of OSN users (e.g., such a group psychological profile could be used to recommend skills/activities/jobs to users based on their profile similarity) and classification of users based on their profiles.\n\nIn this section, we provide an overview of related psychological literature, discuss related work, and highlight several open issues of existing methods. We also highlight the contribution of the present work. Section 3 details the collected dataset and the data mining and feature engineering methodology, and Section 4 presents the design and evaluation of the proposed machine learning predictive model. Finally, we conclude our article and discuss future work in Section 6.\n\nFirst, Please Summarize the paper in 10 points, in easy to read and understand simple English.\nSecond, Explain what the paper does as if I'm 11 years old.\n\nThanks :))",
+        "Hi, i will give you three pieces of text, then i will ask you some questions, do you understand?",
+        "Here is Text 2: Communicating with External Audiences\n\nMany managers believe that they will never have to deal with the press. Often,\nthey regard it with hostility. Most think press relations are entirely the domain\nof their company\u2019s or agency\u2019s public relations department. But in fact, senior\nexecutives say they spend more time on communications than on other tasks,\nand a significant component of that time is devoted to press and public relations.\nJunior managers need to be highly sensitive to press relations for the following\nreasons:\n\u2022 Often, free press can be the best way to acquaint the public with your product or service.\nTo cite only one example, the amount Microsoft spent on advertising Windows\n95 was dwarfed by the value of the free publicity it received from\ninternational news coverage.\n\u2022 Your particular area of expertise may unexpectedly become something your organization\nneeds to promote or explain. Line workers at auto companies have been drafted\nto extol quality improvements in advertisements; accountants may be called\nto the CEO\u2019s office for briefings on a potentially embarrassing news report or\nan upcoming press conference.\n\u2022 Public relations considerations need to be addressed at the beginning, not the end, of a\nplanning process. Business history is replete with examples of companies that\ninvested vast sums to develop products, ideas, or services that couldn\u2019t be sold\nbecause of public resistance to the concept, the configuration, or the public\nimage of the company. General Motors\u2019 Tacos, for example, could be the best\nin the world and still not jump off the shelves.\n\u2022 Junior managers become senior managers who will eventually have to deal with the\npress directly. As both marketers and corporate citizens, organizations have to\nexplain themselves to the public constantly through advertising, press releases,\nand press conferences. Junior managers who understand this aspect of their\nwork are likely to become senior managers faster. 1. A successful manager understands how the press works. Successful managers\ntend to follow the press in general, and how their organization is playing in particular.\nMembers of the press tend to trust companies and individuals with a\ntrack record of accuracy and accessibility. To cite only two examples, both\nJohnson & Johnson and Perrier survived charges of contaminated products because\nthey had a record of reliability and accessibility and addressed the problems\nimmediately. In both cases, and many others, stonewalling would have\nbeen disastrous to the company\u2019s image of wholesomeness and purity. Most\npress stories last only a few days, but they can leave an indelible impression in\nthe public\u2019s mind. Many managers tend to believe they can \u201csnow\u201d the press\nwith their greater expertise, but this strategy rarely works. Most reporters are\nhard-working professionals who will carefully check out an expert assertion or\nwho know someone who can.\n2. A successful manager understands what the press needs. What the press needs\nis a story, and bad news generally sells better than good news. Companies and\nindividuals are most likely to have to deal with the press when something has\ngone wrong. This suggests a couple of lessons. When you have good stories,\ngive them to the press to establish a record of credibility; many media outlets\nwill print or broadcast a press release from a reliable source more or less verbatim.\nConsider how private decisions may look if they should become public.\nIf something has gone wrong, take the initiative in announcing it, explaining it,\nand telling the world how it\u2019s going to be corrected.\n3. A successful manager understands press jargon. Reputable reporters will\nstick to their verbal agreements on how information you provide them is to\nbe used. How you will be quoted depends on the ground rules you establish\nat the beginning of an interview. Deep background means the reporter can\nreflect the information in her story without possible attribution. Background\nmeans that you can be referenced as \u201ca reliable source.\u201d Any other comment,\nhowever apparently casual or social, can be quoted directly and\nattributed.\n4. A successful manager should be able to generate an attention-grabbing, accurate,\nand well-constructed press release. While many managers may not be\nregularly mailing out press releases themselves, most will be contributing to\nthem and need to understand how they work. A good press release is extremely\nformulaic and follows the structure of a good news story:\na. The first paragraph states the main point clearly and emphasizes its newsworthiness.\nFor example: \u201cAcme Corporation announced today that it is\nreleasing the best tire ever available on the world market.\u201d\nb. The second paragraph provides a quote from a reputable source: \u201cAcme\nPresident Rudy Roadrunner said, \u2018Not only does this tire surpass all our\ncompetitors\u2019 in endurance, quality, and safety; it\u2019s also available at a lower\nprice.\u2019 \u201d\nc. The third paragraph provides evidence that the claims made so far are true:\n\u201cIn repeated tests against our competitors . . . \u201d\nd. The remaining paragraphs provide background information on the product, the\ncompany, and Rudy Roadrunner, and they demonstrate a track record of credibility.\nThey may also include testimonials available from respected independent\nsources. Obviously, the formula of an effective press release will vary depending on\nthe nature of the news to be announced. But the pyramid structure suggested by\nthis example always applies: Move from the most important and specific to the\nleast important and most general information. Busy editors often run a press release\nmore or less verbatim and just cut it off when they run out of space. The\neasier you make their jobs, the more likely they are to cover your story.\nOnce you\u2019ve written or contributed to a press release, decide who\u2019s most\nlikely to run it. This can cover the gamut from extremely specialized trade magazines\nto the national or international media. Consider the use of venues other\nthan print and broadcast media as well; perhaps there\u2019s a room on the Internet\nwhere interested parties are likely to gather.\n5. A successful manager understands the role of the press in crisis management.\nThis includes knowing how to provide effective interviews and\nunderstanding when and how to hold a press conference. Certain rules\napply to both:\n\nApplications\na. Identify your central message, make sure you can back it up, and stick to it.\nb. Prepare materials in advance\u2014press releases, statements, supportive\nstudies\u2014that the reporters can take away with them and study or quote later.\nc. Never say more than you know to be true. If you don\u2019t know, say, \u201cI don\u2019t\nhave that information at the moment, but I\u2019ll get it to you as soon as I do\u201d\u2014\nthen follow up.\nd. Make sure your team is behind you. This means making sure not only that\ntop management of a corporation agrees on a message, but also that other\npotential press sources (for example, subordinate employees) have the same\ninformation you\u2019re dispensing to the public, believe it, and are unlikely to\nleak contradictory and embarrassing information.\ne. Provide the press with the most credible and informed access possible. Reporters\nwill always want to get to the top. They\u2019ll be more likely to cover\nthe comments of a CEO or a Cabinet secretary than those of a press agent\nor an underling. But they will understand that a high official may need to\nrefer technical questions to an informed specialist.\nf. Anticipate, and be prepared to respond to, the most difficult questions.\ng. Don\u2019t become hostile or defensive; experienced reporters are experts at\nsmelling anxiety.\nh. Make your answers brief, quotable, and to the point. Rambling and repetition\nare likely to get you into trouble or open new lines of inquiry.\ni. If you\u2019re facing a problem you\u2019ve caused, however inadvertently, be prepared\nto acknowledge\n\nAre you ready for text 3?",
+        "Here is Text 3: Diversity and Intercultural Communication \n\nGenerally, the best answer to these questions is yes, but it always depends on the personal as well as the business aspects of your relationship. One good rule of thumb: When the other person gives\nyou an opening, pursue it, and build on your mutual experience.\nThis issue comes up even more in international communication. As companies\nfrom manufacturers to media conglomerates become increasingly global, managers\nneed to understand the norms of other cultures. Although English is on the verge of\nbecoming the international language, standards of behavior and social interaction\nvary greatly between the United States and England, let alone between, say, France\nand Japan. In one country an invitation to dinner may be considered an expected\npoliteness, while in another, it may be an invasion of a colleague\u2019s private time.\nAsking about someone\u2019s family may be absolutely required in one culture and offensively\nintrusive in another.\nNo textbook can cover all such contingencies; one good rule if you\u2019re not sure\nmay be the trial lawyer\u2019s: Don\u2019t ask a question to which you don\u2019t already know the\nanswer. Another, and sometimes contradictory, rule is: Be frank about your cultural\nconfusion. Your colleague likely will have been in the same situation himself and\nwill be happy to help out. Finally, do your research; you\u2019re likely to have a friend or\ncoworker who knows the terrain better than you do. Our purpose here is to sensitize\nmanagers to their increasing need to understand the norms of cultures other than\ntheir own. (For a case addressing the special features of international communication,\nsee International Oil later in this chapter.)\nThe opportunities for cultural confusion\u2014personal, commercial, ethical, and\nlinguistic\u2014are almost endless. Imagine marketing a Chevy Nova in Hispanic countries,\nwhere \u201cno va\u201d means \u201cit doesn\u2019t run.\u201d Many products that are perfectly safe to\nmarket in first-world countries raise ethical problems when sold in developing\ncountries\u2014infant baby formula, for example, which if mixed with contaminated\nwater can cause death. Working in other cultures means understanding your hosts\u2019\nconceptions of greetings, timing, hygiene, negotiation, agreement, politeness, personal\nspace, gesture, meal etiquette, and closure.\nWhile English has essentially become the international language, it\u2019s important\nto remember that there are many Englishes. A joke in one form of English can be a\ndeadly insult in another. Although it may seem too obvious to emphasize, you must\nunderstand the cultural norms and language use of people from other cultures before\nyou can communicate effectively with them. This is true even if they are, say,\nthe South American employees of your Canadian company. A bribe in one culture\ncan be a thoughtful gift in another.\nA recent article by Sydel Sokuvitz (Business Communication Quarterly, New\nYork, March, 2002) suggests some principles for conducting successful intercultural\nbusiness communication. Sokuvitz first describes the special challenges global\nmanagers face, including:\nCoping with a range of tensions that arise out of internationally dispersed activities,\nThe challenges of maintaining coordinated activities across time-zones, cultural\nboundaries, and different countries\u2019 laws, and\nThe difficulties posed when the right medium for your message in one culture\nmay be wrong in another.\nDrawing on a range of research in the field, Sokuvitz comes up with several\nprovocative conclusions:\nExcessive dependence on technological communication such as E-mail can result\nin problems for both communication and productivity.\nFace-to-face meetings with colleagues from other cultures are critical to achieving\neffective communication.\nStudying with students from other cultures is critical to preparing a manager\nfor working in the increasingly globalized economy.\nSokuvitz cites the following example from an article by Fernandez-Aroaz\n(\u201cHiring without Firing,\u201d Harvard Business Review, 1999):\nA U.S.-based telecommunications company was seeking a CEO for its new division\nin Latin America. An international search was conducted, and a veteran was\nhired, someone known as an effective manager and marketing expert. \u201cBut his run\nlasted less than a year and was nothing short of a disaster. The simple reason was\nthat he lacked the two skills that the job really required: negotiation and cross-cultural\nsensitivity.\u201d\nEventually the company was saved from near-bankruptcy by bringing in a\nnew CEO who was a native Latin American with work experience in the U.S. His\nability to bridge cultural differences is credited with saving the company.\nCommunications between headquarters and subsidiaries is only one example\nof the challenges posed by globalization. Companies in one country are under increasing\nsocial pressure to take responsibility for the behavior of their subcontractors\nin other countries. Recently, for example, Nike suffered adverse publicity because\nof the work practices of shoe manufacturers it employs in Asia.\nThe successful manager of the future increasingly will be required to be a citizen\nof the world. While electronic communication may work fine for conveying information\nor directions, there is no substitute for \u201cspeaking the language\u201d of the\npeople with whom you\u2019re trying to communicate.\n\nAre you ready to answer some questions on text 1, text 2 and text 3?",
+        'pragma solidity ^0.4.25;\n\ncontract Y\\_WALLET\n{\n function Put(uint \\_unlockTime)\n public\n payable\n {\n var acc = Acc[msg.sender];\n acc.balance += msg.value;\n acc.unlockTime = \\_unlockTime>now?\\_unlockTime:now;\n LogFile.AddMessage(msg.sender,msg.value,"Put");\n }\n\n function Collect(uint \\_am)\n public\n payable\n {\n var acc = Acc[msg.sender];\n if( acc.balance>=MinSum && acc.balance>=\\_am && now>acc.unlockTime)\n {\n if(msg.sender.call.value(\\_am)())\n {\n acc.balance-=\\_am;\n LogFile.AddMessage(msg.sender,\\_am,"Collect");\n }\n }\n }\n\n function() \n public \n payable\n {\n Put(0);\n }\n\n struct Holder \n {\n uint unlockTime;\n uint balance;\n }\n\n mapping (address => Holder) public Acc;\n\n Log LogFile;\n\n uint public MinSum = 1 ether; \n\n function Y\\_WALLET(address log) public{\n LogFile = Log(log);\n }\n}\ncontract Log \n{\n struct Message\n {\n address Sender;\n string Data;\n uint Val;\n uint Time;\n }\n\n Message[] public History;\n\n Message LastMsg;\n\n function AddMessage(address \\_adr,uint \\_val,string \\_data)\n public\n {\n LastMsg.Sender = \\_adr;\n LastMsg.Time = now;\n LastMsg.Val = \\_val;\n LastMsg.Data = \\_data;\n History.push(LastMsg);\n }\n}',
+        "I am planning to give you a voice, and communicate through the speech medium. I need a speech recognizer, a wake call detector, and a speech synthesizer for your voice. Suggest a python script utilizing existing libraries to achieves the goal.",
+        "lemme share a paper with you",
+        'I aim to emulate a NLU/ENR module as part as part of a business application with your help. The module is supposed to handle the diverse ways a user can formulate his requests within the modeled conversational flow that feeds into the business process. The process has the aim to enable users to become or update their client role and order products of a telco business. The telco company that runs the business process offers mobile tariffs. Mobile tariffs have can have between one and 5 sim cards. Each booked sim cards enables the user to optionally book a smartphone for that card. Depending on the tariff, the chosen smartphones (if any) and the kind of sim cards (adult, child) the price will adapt. Please suggest a set of NLU / ENR methods that you could emulate to facilitate the use case. In the following I will input utterances and statements on how the system running the conversational flow should handle the utterance within the conversational flow. Please provide possible calls to an imaginary API that you could simulate to facilitate the NLU/ENR requirements layed out by my statements. On Subtasks that are recognized as not directly related to NLU/NER be very brief. Please suggest NLU / NER Operations now for the first of a few utterances: "Hi I want to upgrade my current tariff and get a new smartphone". The utterance should make the system recognize that the utterance can be handled as part of the business process. It should recognize that the user apparently already a client and it should continue the conversation by trying to identify him and metadata on his current tariff. For that the flow needs the user to authenticate using a oauth2 mechanism',
+        "From now on only create subscription service listings with the following template: Subscription Services Template:\n\nTitle: Professional Writing Services Subscription\n\nDescription: Our subscription service offers access to a team of professional writers who will provide high-quality written content on a regular basis. Choose from one of our three plans to suit your needs and budget.\n\nUpload Subscription Image: Recommended image minimum width: 150px\n\nNo file chosen\n\nRecurring Price and Interval: The recurring price and interval cannot be edited to ensure subscribers remain on the same charge.\n\nPlan 1:\nPlan name: Basic\nThe recurring price is USD 75.00 and will be charged periodically at every 1 month\nPlan description: This plan includes access to a professional writer who will provide one piece of written content per month. Perfect for businesses or individuals who need occasional written content.\n\nPlan Image: Display a small image to represent this plan to customers\n\nTrial Period: Enable trial period\nAssign Digital Product Files: Assign digital products for subscribers\n\nPlan 2:\nPlan name: Pro\nThe recurring price is USD 500.00 and will be charged periodically at every 1 month\nPlan description: This plan includes access to a team of professional writers who will provide up to five pieces of written content per month. Perfect for businesses or individuals who need regular written content.\n\nPlan Image: Display a small image to represent this plan to customers\n\nTrial Period: Enable trial period\nAssign Digital Product Files: Assign digital products for subscribers\n\nPlan 3:\nPlan name: Premium (Bundle of 20 / 1,500 words)\nThe recurring price is USD 1000.00 and will be charged periodically at every 1 month\nPlan description: This plan includes access to a team of professional writers who will provide up to 20 pieces of written content per month. Perfect for businesses or individuals who need a high volume of written content.\n\nPlan Image: Display a small image to represent this plan to customers\n\nTrial Period: Enable trial period\nAssign Digital Product Files: Assign digital products for subscribers",
+        "Hello",
+        "I am launching an Etsy shop with a Printful integration for drop shipping my designs on specific products. I am looking for ways to differentiate beyond the designs. You are an expert on Etsy audiences. Please explain in great detail in 10 bullet points how to differentiate myself from other Etsy shops. I am looking for more obscure ideas here.",
+        "How to get a job as a LMFT therapist in the US as an international student?",
+        "Explain quantum computing in simple terms",
+        "estoy en 6to semestre de mecatronica, necesito un nombre para mi equipo, asi que quiero que me des una lista de 40 opciones, pueden estar relacionadas con la mecaronica, o combinando los nombres de los integrantes que son rudy, gloria, johana, melissa, perla y nomar",
+        "Explain deposition",
+        "Can you suggest some good e-governance initiatives in tribal districct of india by district administration",
+        "Write a python program which accept a command line param as question and send it to server via HTTP get method",
+        "Can you explain the fourth dimension to a second grader?",
+        "I have an interview about product speccing with the company Weekend Health. Give me an example of a question they might ask with regards about a new feature",
+        "arduino uno adalah",
+        "how edit array which is in object",
+        "how can my software company use Microsoft ENTRA to verify the identity of a user before accessing the software?",
+        "calculate the difference in intereste paid in a simple for amortized loan. terms: 125,000 loan, 3.25% interest over 30 years.",
+        "can i use spring state machine and workflow together and is it justified?",
+        'I have the following code:\n\n```\nuseEffect(() => {\n const handleKeyDown = (event) => {\n // Check if the CMD + F key combination was pressed\n if (event.key === "f" && event.metaKey) {\n event.preventDefault();\n\n setIsShown(true);\n }\n\n window.addEventListener("keydown", handleKeyDown);\n\n return () => {\n window.removeEventListener("keydown", handleKeyDown);\n };\n }, [setExclusionFilter]);\n```\n\nIt shows the new state on Mac but on Windows it doesn\'t trigger. How can I support windows?',
+        "What is the best marketing tactics for local small businesses?",
+        "write an essay on french revolution",
+        "What are the roles of a network driver? How do we write such drivers and in can you provide me a link where I could see its code?",
+        "Are you familiar with the SAS programming language?",
+        "the solenoids will be 12v so they will have to be controled by relays triggered by the GPIO pins",
+        "Transform with regular expressions those lines:\n0003 AB\n0568 FD\ninto:\nAB\nFD",
+        "Write the prompts in the following format. First sentence establishes a situation. Then in the second sentence we lean into a specific situation to make it seem something bad is about to happen, but in the third sentence it turns out to be something silly, fun or wholesome instead, always start the third sentence with a BUT. Some examples below\n\n-A hydra is hypnotizing an orc. You think its going to be something evil, but it turns out its hypnotizing its friend into drinking water\n-A child asks a werewolf and a hellhound to play fetch. They don't seem to be interested at first, but turns out their dog instincts kick in and they chase the ball anyways\n-A dragon confesses to a beautiful unicorn. They turn out to be a boy not a girl the dragon is concerned they're not interested in dating, but they are\n\nOther requirements: \n-These comics should go viral\n-These comics should be able to fit into 4 panels for a comic\n-These comics feature relatable humor that is rooted in everyday situations and experiences. \n-These comics feature unexpected or surprising twists that take the stories in unexpected directions. \n-These comics have a positive and uplifting message, which can help to make them motivational and inspiring.\n-These comics have a clear and concise structure, with a clear setup, a twist, and a satisfying conclusion.\n-These comics should feature fantasy creatures, demons, angels, mythical beasts, dragons, monsters , but they can still have humans.",
+        "How can we improve this comic to be simpler and funnier?\n\n[We see that this is a small reading club for woodland creatures. Make them all nice and cute, very winnie the pooh-esque, lol. The two characters that speak are animals, make Red into a herbivore race, like a rabbit or something, pink should be a small carnivore like a cat or badger? Red is confused, and red is excited]\nKnock Knock\nPink:Who\u2019s that?\nRed: Maybe a new member for our book club!\n\n[Panics as she sees a dragon licking their lips behind the curtain]\nRed: It\u2019s a dragon, run for your lives everyone!\n\n[Dragon mom is outside their home, looking dragon-eque but also waving her hands chibi cute apologetically, she\u2019s clearly a little embarrassed by the situation. Red looks at her suspiciously ]\nDragon:I\u2019m not here to eat anyone, I uh\u2026 heard you had a book club?\nRed: Uh\u2026yes\n\n[Dragon looks very excited and welcome, Pink seems like she likes the book, red looks a little grossed out ]\nDragon: Awesome, it's nice to meet you! I brought my favorite book too!\nPink: What a lovely book!\nRed: Ugh I\u2019ll pass on reading that.",
+        "Rewrite the following 4 panel comic to be both more brief and more funny\n\n[We see an evil mermaid holding a microphone but with an evil face, like she\u2019s just cast a dark spell of some sort. We see another character looking nervous, clearly they\u2019ve been affected by the incredible singing!]\nMermaid: You\u2019ve lost! Give up & spare us both the trouble!\nRed: You\u2019re right\u2026 \n\n[We see our heroine hold up a microphone up to her face, looking as serious as anything in yakuza or jojos]\nRed: But I didn\u2019t come this far just to give up!\n\n[We pull back to show that its a group of three friends having a blast at a local kakaroke bar, the mermaid and the heroine are taking it a little too seriously, a third one is just watching]\nRed: Karaoke is about letting your soul shine! I\u2019m giving it my all or die trying!\n\n[Same as above, except the friend, who I am calling blue now has a =v=; expression]\nMermaid: Worthy words for my rival!\nBlue: Girls, you need to chill. \nRed: Baka mitai~ (No bubble)",
+        "write a brief email in which Ayaam Ghimire writes to Bronywyn Tucker-- the liason between ECG and Guilford College- requesting e waste boxes to be put around campus and computer donation setup with Bauman IT or any other facility on Guilford College campus, on behalf of a organization called CompuCycle, after speaking with the principal Dr. Kash",
+        "I'm writing a software for conference calls.\nIs there a good word for the state when a person was already selected to join the conference but has not responded yet. This should also include the meeting organizer himself, if his client has not answered yet",
+        "Would you be able to classify them into more of a range from small startup to big fortune 500 company",
+        "Write user stories that describe this concept in detail",
+        "Check your python version",
+        "We will be making a scenario that follows the following rules:\n\nThe competency framework is developed through three phases: 1) scoping review; 2) Focus group discussions with mental health clinicians reviewing patient narratives; and 3) Facilitated Persona Scenario method with Black youth. Moreover, the project adopts a co-design approach and convenes a Knowledge User Panel. The panel will be involved in all phases of the competency framework development as they will review findings from the scoping review and focus groups. \n\nFocus group with mental health clinicians \n Mental health clinicians (i.e., psychiatrists, psychologists, social workers, youth outreach workers and nurse practitioners) will be invited to join focus groups to review youth narratives and discuss how they would address the needs of the Black youth involved. The youth narratives will be generated through collecting stories from social media and through an online survey. The survey will ask about young people's experiences with mental health conditions, their use of mental health services, and their suggestions for how to improve mental health care for young people. The online survey will collect stories anonymously. Anyone who submits a story through the survey will be redirected to a list of resources. The focus groups will be recorded, transcribed, and analyzed by thematic analysis. The focus groups will continue until thematic saturation.\n\nPhase 3: Persona Scenario method with Black youth\n Black youth will be invited to focus groups (or one-on-one interviews, if requested) using persona scenario methods. The findings from the focus groups with mental health clinicians will be used to create clinician personas, including information about their motivations, challenges and describe the different ways in which the clinician might interact with the Black youth based on youth narratives. Black youth will be asked to share their perspectives and preferred clinician responses. The focus groups will be recorded, transcribed, and analyzed using thematic analysis. We will continue to hold focus groups until thematic saturation.\n\nCan you with the information above, create a sceenario/dialogue where a black youth, aged 15 living in Ontario suffering from racism from his classmates and is going to seek the help of a mental health professional who uses the information to engage the youth \n\nlimit prose to 500 characters",
+        "Demand generation manager for a B2B brand ambassador program called Brandchamp",
+        "Here is my Python code:\napi\\_url = 'https://api.yelp.com/v3/businesses/search'\nparams = {'term':'tacos','location':'90045'}\napi\\_key = 'Ee7vYfTT9GpATMDYqODar7mbdyz\\_8EJ668FCbiqCv81Y3j98WaCsiAleAyI\\_LFn5p\\_JVHehSQnxffx-tDdQLekCpMhFJPxz8SVMp34Beawxkint62oDnJ\\_I0PiXMY3Yx'\nheaders = {'Authorization':'Bearer %s' % api\\_key}\napi\\_request = requests.get(api.\\_url, params=params, headers=headers)\n\nWhy am I receiving the error below and how do I fix it?\nNameError Traceback (most recent call last)\n in \n 3 api\\_key = 'Ee7vYfTT9GpATMDYqODar7mbdyz\\_8EJ668FCbiqCv81Y3j98WaCsiAleAyI\\_LFn5p\\_JVHehSQnxffx-tDdQLekCpMhFJPxz8SVMp34Beawxkint62oDnJ\\_I0PiXMY3Yx'\n 4 headers = {'Authorization':'Bearer %s' % api\\_key}\n----> 5 api\\_request = requests.get(api.\\_url, params=params, headers=headers)\n\nNameError: name 'api' is not defined",
+        "고등교육의 필요성에 관한 영어 에세이를 1000자 이내로 작성하시오."
+        "Which hero is the best in Heroes of Might and Magic 3?",
+        "Use C# to get the current YouTube thumbnail and convert it to Base64.",
+        "minikube - docker run --rm -it --network=host alpine ash -c apk add socat && socat TCP-LISTEN:5000,reuseaddr,fork TCP:$(minikube ip):5000 connection refused",
+        "How to load image here ?",
+    ]
+
+    responses = await generate_multi(flash_llama, prompts, max_new_tokens=10)
+
+    assert len(responses) == len(prompts)
+    outputs = [r.choices[0].message.content for r in responses]
+    assert outputs == [
+        "Jeff Walker's Product Launch Formula is a comprehensive system",
+        "Here are three key indicators to determine if a customer",
+        "You can use the `String.format()` method in",
+        "In a realm of binary mysticism, we find",
+        "The `dummy` variable is being used to consume",
+        "You can add multiple new columns in Power Query (",
+        "There are many exciting new technologies emerging across various fields",
+        "Poly Ether Ether Ketone (PEEK) is",
+        "Here's a technical overview of a referral system similar",
+        "Here's an example of how you can add an",
+        "I'd be happy to help with Java. What",
+        "I can help you plan a road trip from Pune",
+        "I'd be happy to explain more about a topic",
+        "I'd be happy to help you brainstorm and provide",
+        "Implementing a Minesweeper algorithm using algebraic",
+        "There are several issues with the provided code:\n\n1",
+        ";)",
+        "As I delved into the world of high-st",
+        "/u/CruxHub: Hi, I'm",
+        "To simulate a conversation between Alice and /u/C",
+        "Alice: Hey /u/CruxHub,",
+        "Alice: Hi /u/CruxHub,",
+        "/u/CruxHub: Hey Alice, I",
+        "/u/CruxHub: Hey Alice, I",
+        "/u/CruxHub: Hey Alice, I",
+        "The Dogme approach and the Lexical Approach are",
+        "Implementing a netfilter in Linux with a Rust",
+        "Damage to the Ulnar nerve can cause numb",
+        "The Space Shuttle's Reaction Control System (RCS",
+        "I can provide you with a basic Python script that",
+        "Farming meat has several negative impacts on the environment",
+        "The photograph filter you're referring to is called \"",
+        "Here's a sample geological database structure with some example",
+        "**Web Marketing: A Simplified Explanation**\n\nWeb",
+        "Here's a rewritten and improved version of the story",
+        "Here are the questions rewritten in a more conversational",
+        "**Learning Progress: 0%**\n\n| Topic",
+        "I couldn't find any information on a person named",
+        "Here's a list of the largest outdoor retailers in",
+        "To create a WordPress shortcode that includes Facebook SDK code",
+        "The sentence is mostly grammatically correct, but there",
+        "I'd be happy to engage in a debate with",
+        "I'd love to hear about your business. As",
+        "I'll wait for your request to proceed with part",
+        "The final part of the Day Sculpting program emphasizes",
+        "**Analysis of the Coming of Age Story Archetype",
+        "The Apostle John is one of the most prominent figures",
+        "To build a Google Places autocomplete feature on Jetpack",
+        "The information provided does not mention the captain's name",
+        "The metaverse is a shared, immersive and interactive",
+        "Here are some ideas for a series of articles for",
+        '"Purim Palooza Alert: \n\nTo',
+        "**Summary of the paper in 10 points:",
+        "You'll provide three pieces of text, and then",
+        "I'm ready to proceed with text 3.",
+        "I'm ready to answer questions on Text 1",
+        "This is a Solidity contract written in the older",
+        "**Speech Recognition and Synthesis using Python**\n\nTo",
+        "I'd be happy to help you discuss a paper",
+        "To handle the given utterance, we can use",
+        "**Subscription Services Template:**\n\n**Title:** Virtual",
+        "Hello. How can I assist you today?",
+        "Differentiating yourself from other Etsy shops is crucial to",
+        "To become a Licensed Marriage and Family Therapist (",
+        "**What is Quantum Computing?**\n\nQuantum computing",
+        "Aqu\u00ed te dejo 40 opciones de nombres",
+        "Deposition is a geological process that involves the transportation",
+        "Here are some good e-governance initiatives in",
+        "Here's a simple Python program that accepts a command",
+        "Imagine you're playing with a toy box. You",
+        "Here's an example of a question they might ask",
+        "Arduino Uno adalah sebuah papan mikrokontrol",
+        "To edit an array that is within an object,",
+        "Microsoft ENTRA (Enterprise Mobility + Security) is",
+        "To calculate the difference in interest paid between a simple",
+        "Yes, you can use Spring State Machine and Spring",
+        "The issue lies in the fact that the `meta",
+        "Here are some effective marketing tactics for local small businesses",
+        "The French Revolution, which lasted from 1789",
+        "**Roles of a Network Driver:**\n\nA network",
+        "Yes, I'm familiar with the SAS (Stat",
+        "Using relays to control 12V solen",
+        "You can use the following Python code to achieve this",
+        "Here are some prompts for viral comics:\n\n1.",
+        "To simplify and make the comic funnier, consider",
+        "Here's a rewritten version of the 4-panel",
+        "Subject: Request for E-Waste Collection and Computer",
+        "In the context of conference calls, the state you",
+        "I can provide a general classification of companies based on",
+        "Here are some user stories that describe the concept in",
+        "You can check your Python version by running the following",
+        "**Scenario:**\n\n15-year-old Black youth,",
+        "As a Demand Generation Manager for a B2B",
+        "The error is due to a typo in your code",
+        "고등교육의 필요성에 관한 영어 에",
+        "Here's a simple C# program that uses the",
+        'The error message "connection refused" indicates that the',
+        "To load an image, you can use various methods",
+    ]
+    assert responses == generous_response_snapshot
diff --git a/integration-tests/models/test_flash_llama_prefix_flashdecoding.py b/integration-tests/models/test_flash_llama_prefix_flashdecoding.py
new file mode 100644
index 00000000..73d397bd
--- /dev/null
+++ b/integration-tests/models/test_flash_llama_prefix_flashdecoding.py
@@ -0,0 +1,229 @@
+import pytest
+
+
+@pytest.fixture(scope="module")
+def flash_llama_handle_fd(launcher):
+    with launcher(
+        "meta-llama/Meta-Llama-3.1-8B-Instruct", num_shard=2, attention="flashdecoding"
+    ) as handle:
+        yield handle
+
+
+@pytest.fixture(scope="module")
+async def flash_llama_fd(flash_llama_handle_fd):
+    await flash_llama_handle_fd.health(300)
+    return flash_llama_handle_fd.client
+
+
+@pytest.mark.asyncio
+@pytest.mark.private
+async def test_flash_llama_flashdecoding(
+    flash_llama_fd, generate_multi, generous_response_snapshot
+):
+    prompts = [
+        "Summarize the main ideas of Jeff Walker's Product Launch Formula into bullet points as it pertains to a growth marketing agency implementing these strategies and tactics for their clients...",
+        "How to tell if a customer segment is well segmented? In 3 bullet points.",
+        'In Java, I want to replace string like "This is a new {object} at {place}" with a Map, {object: "student", "point 3, 4"}, and get a result "This is a new student at point 3, 4". How can I do?',
+        "Metaphorical language is also used to describe the various addressing modes of the instructions. Grandiose language to express their excitement and admiration for the functionality of the instructions being described. Now, rewrite this with more perplexity:\n\nJMP ABCD\nMOV AX, [BX+SI]\nMOV AX, [100]\nMOV AX, [BX]\nMOV AX, [BX\\*2+SI]\nMOV AX, BX\nMOV AX, 7",
+        'I have the following C++ function: \nvoid add\\_player(vector& players)\n{\n string player\\_name;\n string player\\_class;\n string dummy;\n PlayerClass pc;\n string player\\_sex;\n int player\\_gold;\n\n cout << " Create a Mage, Warrior, Bowman, or Thief" << endl;\n\n cout << "Name: ";\n getline(cin, player\\_name);\n\n cout << "Class: ";\n getline(cin, player\\_class);\n pc = get\\_player\\_class\\_from\\_string(player\\_class);\n while (pc == PlayerClass::InvalidPlayerClass)\n {\n cout << " Invalid class, try again" << endl;\n cout << "Class: ";\n getline(cin, player\\_class);\n pc = get\\_player\\_class\\_from\\_string(player\\_class);\n }\n\n cout << "Sex: ";\n getline(cin, player\\_sex);\n\n cout << "Gold: ";\n cin >> player\\_gold;\n getline(cin, dummy); //consume newline\n\n GamePlayer new\\_player;\n new\\_player.name = player\\_name;\n new\\_player.occupation = pc;\n new\\_player.gender = player\\_sex;\n new\\_player.gold = player\\_gold;\n\n //add to vector\n players.push\\_back(new\\_player);\n\n //add to file\n write\\_players\\_file(players);\n}\nCan you explain to me how the dummy variable is being used?',
+        "how do I add multiple new columns in m for power query or power bi?",
+        "Sure, I can do that. What new technology would you like me to review?",
+        "Poly Ether Ether Ketone",
+        'can you design a referral system similar on how dropbox did? I need a technical overview on how it should work, instead of free space we use the generic term "credits" where users can get more credits for every 3 friends they recommend.',
+        "Java add to the arraylist of a class type",
+        "this is not less code this is java",
+        "I want to do a road trip from Pune to Gujarat. Me and my wife will be travelling and we dont prefer very long driving sessions. Can you suggest a plan starting from Thursday early morning and ending in Pune on Sunday late night.",
+        "explane more",
+        "what do you think about this for a start up idea:",
+        "how could i implement a minesweeper algorithm that utilises algebraic topology to solve boards?",
+        "# Import the necessary packages\nfrom gudhi import SimplexTree\nfrom gudhi.persistent\\_homology import PersistentHomology\n\n# Define a function to compute the persistent homology of a Minesweeper game board\ndef minesweeper\\_homology(board):\n # Create a simplicial complex for the game board\n st = SimplexTree()\n\n # Add the points on the board to the simplicial complex\n for i in range(len(board)):\n for j in range(len(board[0])):\n st.insert([i, j], filtration=board[i][j])\n\n # Compute the persistent homology of the game board\n ph = PersistentHomology()\n ph.build(st)\n\n # Return the persistent homology diagram\n return ph.persistence()\n\n# Define a function to solve a Minesweeper game board using persistent homology\ndef minesweeper\\_solver(board):\n # Compute the persistent homology of the game board\n homology = minesweeper\\_homology(board)\n\n # Use the persistent homology to determine the locations of the mines\n # (this part would require some mathematical reasoning and programming)\n mines = []\n for h in homology:\n if h[1] - h[0] == 1: # if the hole persists for one filtration value\n mines.append(h[0]) # then it corresponds to a mine\n\n # Use the information about the mines to solve the game\n # (this part would require some programming)\n for mine in mines:\n i, j = mine # extract the coordinates of the mine\n board[i][j] = -1 # mark the mine on the board\n # (other code to solve the game)\n\n \nwhat is missing here?",
+        "You are now an imaginary expert business investigator. I am going to send you many rows of data. Each batch of row's will be sent to you and you may only reply \"Received.\" Save any analysis or insights for after you've received all of the data and I've told you \"Let's Begin.\" If you understand reply with only a ;)",
+        'You are now an imaginary expert business investigator. Tell the story of this batch of data in the form of a narrative story about the companies in the "Entity Name" column: \n\nBatch of data #1: Entity Name Purpose / Source\n101 PC HOLDINGS LLC Holding company for Penthouse C at the Setai Miami Beach (folio: 02-3234-153-1160)\n11 STAR ISLAND LLC Holding company for 10 STAR ISLAND DR, MIAMI BEACH, FL 33139 (folio: 02-4204-001-0100, 02-4204-001-0110) (lots 10, 11 and 12 of Star Island)\n117 EAST PARK AVENUE, LLC Holding company for 117 E. PARK AVE, LIBERTYVILLE, IL (PIN: 11-21-212-046-0000); subsequently sold.\n1201 BRICKELL BAY, LLC Holding company for 1201 BRICKELL BAY DR, MIAMI, FL (folio no: 141390710010)\n1221 BRICKELL, LLC Holding company for 1221 BRICKELL AVE, 155 SE 13 ST, 165 SE 13 ST, 175 SE 13 ST, and 185 SE 13 ST, MIAMI, FL (folio: 01-4139-035-0010)\n1221 BRICKELL HOLDINGS LLC Holding company for 1221 BRICKELL, LLC\n1229 PARK WEST AVENUE, LLC Holding company for 1229 W. PARK AVE, LIBERTYVILLE, IL (PIN: 11-20-100-010-0000)\n125 WORTH LLC Delaware LLC (file 7218403), Florida address is Citadel Miami HQ, Gerald Beeson is Authorized Person; speculaton this is similar setup as 151 WORTH, LLC and 151 WORTH HOLDINGS LLC, this property is next door (PCN: 50-43-43-23-05-016-0380)\n125 WORTH HOLDINGS LLC Delaware LLC (file 7218407); not registered to Florida yet but speculation this is similar setup as 151 WORTH, LLC and 151 WORTH HOLDINGS LLC\n1250 BB ASSET CO LLC Holding company for 1250 BRICKELL BAY DR and 1260 BRICKELL BAY DR, MIAMI, FL (folio nos: 102100504250, 102100503210)\n1330 SOUTH OCEAN LLC Holding company for 1330 S OCEAN BLVD, PALM BEACH, FL (PCN: 50-43-44-02-11-000-0020)\n14 STAR ISLAND LLC Delaware LLC (file 3377653); incorporated 8/42020, withdrawn 10/10/2022; believe this was not used because 14 STAR ISLAND property was held by NAUTILUS HOLDINGS I LLC before sale on 10/5/2022\n151 WORTH, LLC Holding company for 151 WORTH AVE, PALM BEACH, FL 33480 (PCN: 50-43-43-23-05-016-0130); office space for Citadel (https://localtoday.news/fl/citadel-moves-into-palm-beachs-former-neiman-marcus-building-4821.html); sole member is 151 WORTH HOLDINGS LLC\n151 WORTH HOLDINGS LLC Holding company for 151 WORTH, LLC\n16 WILLOW HOLDINGS LLC f/k/a PVNAH LLC Holding company for S WILLOW COURT, ASPEN, CO (Parcel: 273511309030); see Pitkin Co. reception # 623002, Delaware certificate showing name change 9/1/2015\n190 PFISTER HOLDINGS LLC f/k/a AH2013 HOLDINGS LLC Holding company for 190 PFISTER DR, ASPEN, CO (parcel: 273511309029); see Pitkin Co.reception # 623000, Delaware certificate showing name change 9/1/2015\n196 PFISTER HOLDINGS LLC Holding company for 196 PFISTER DR, ASPEN, CO (parcel: 273511309028); see Pitkin Co. reception # 623501, statement of authority show KP HOLDINGS LLC as sole membe\n1ALPH LLC See ALPH LLC\n1BUSINESS GROUP LLC See BUSINESS GROUP LLC\n1GFS DESIGN LLC See GFS DESIGN LLC\n1GFS LLC See GFS LLC\n1MEDIA HOLDINGS LLC See MEDIA HOLDINGS LLC\n23174 NE 41ST PATH LLC Holding company for 23174 NE 41ST PATH #12, OKEECHOBEE, FL 34972 (Parcel: 1-01-35-35-0020-00000-0120); part of Pine Creek Sporting Club (www.pinecreeksportingclub.com) includes horse, shooting sports; sole member is KP HOLDINGS L.L.C.\n3031 BRICKELL LLC Holding company for 3031 BRICKELL AVE, MIAMI FL 33129 (Folio: 01-4139-001-2700); Sole member is KP HOLDINGS L.L.C.\n31 WILLOW HOLDINGS LLC f/k/a AP HOLDINGS I LLC Holding company for 31 NORTH WILLOW COURT, ASPEN, CO (Parcel: 273511309019); sold 7/6/2017; see Pitkin Co. reception # 623001, Delaware certificate showing name change 9/1/2015\n650 CASUARINA LLC Holding company for 650 CASUARINA CONCOURSE CORAL GABLES, FL (folio: 03-4132-019-0060) https://www.bizjournals.com/southflorida/news/2022/05/27/650-casuarina-concourse-coral-gables-sold.html\n650 MEADOW LANE 1 LP Holding company for 650 MEADOW LANE, VILLAGE OF SOUTHAMPTON, NY (Parcel ID 7478) (https://archive.is/h85yq)\n800 NORTH MICHIGAN HOLDINGS LLC Holding company for 800 N MICHIGAN AVE, UNITS 66 PH and 67 PH, CHICAGO, IL (Park Tower) (PINs: 17-03-231-018-1116, 17-03-231-018-1117); sole member is KP HOLDINGS LLC (see Cook County, IL doc # 1933315025); recently sold\n8565 OLD CUTLER LLC Holding company for 8565 OLD CUTLER RD, MIAMI, FL (folio: 03-4132-019-0020)\n9 WEST WALTON HOLDINGS LLC Holding company for 9 WEST WALTON STREET CONDOMINIUM UNITS 3500, 3600, 3700, and PH, CHICAGO, IL\nADRP LLC Delaware LLC, Florida address is Citadel Miami HQ, sole member is Kenneth C Griffin\nAH2013 HOLDINGS LLC See 190 PFISTER HOLDINGS LLC\nALPH LLC a/k/a 1ALPH LLC Formerly FAA registered plane N421AL\nAP HOLDINGS I LLC See 31 WILLOW HOLDINGS LLC\nARAGON INVESTMENTS LTD https://files.brokercheck.finra.org/firm/firm\\_45631.pdf\nASHLER CAPITAL LLC https://adviserinfo.sec.gov/firm/summary/148826\nASHLER CAPITAL MASTER FUND LTD https://www.sec.gov/Archives/edgar/data/1003078/000114420418014250/tv488357\\_sc13g.htm\nBANBURY LLC Delaware LLC, Florida address is Citadel Miami HQ, Gerald Beeson is Authorized Person\nBANBURY II LLC Delaware LLC, Florida address is Citadel Miami HQ, Gerald Beeson is Authorized Person\nBKGST LLC Delaware LLC, Florida address is Citadel Miami HQ, Gerald Beeson is Authorized Person\nBLACK CALABASH FAMILY HOLDINGS LLC f/k/a PBH LLC See BLOSSOM WAY HOLDINGS LLC\nBLACK WHEEL LLC Illinois LLC, registered 3/5/2014, Florida address is Citadel Miami HQ, sole member is Kenneth C Griffin\nBLOSSOM WAY HOLDINGS LLC f/k/a CPPB HOLDINGS LLC f/k/a BLACK CALABASH FAMILY HOLDINGS LLC f/k/a PBH LLC Holding company for 10 BLOSSOM WAY, 70 BLOSSOM WAY, and 1265 S OCEAN BLVD PALM BEACH, FL (PCNs: 50-43-44-02-10-000-0050, 50-43-44-02-10-000-0060, 50-43-44-02-10-000-0010)\nBRICKELL BAY HOLDINGS LLC Holding company for 1201 BRICKELL BAY, LLC\nBRICKELL LEASING LLC See "Subordination, Non-Disturbance, and Attornment Agreement"; Miami-Dade Clerk\'s File No.: 2022 R 938960, Group: 1. Kenneth C Griffin is sole member.\nCAAM MANAGEMENT LLC https://www.sec.gov/Archives/edgar/data/1027745/000114420408050200/v124853\\_sc13g.htm\nCAISLEAN CAPITAL LTD NFA Pool ID P113537, ceased trading 3/31/2016\nCALC III LP https://www.sec.gov/edgar/browse/?CIK=1582652\nCALC IV LP https://www.sec.gov/edgar/browse/?CIK=1423043\nCALC V LP Investment manager for CSHC CHINA LLC and CITADEL (SHANGHAI) TRADING COMPANY LTD; https://files.brokercheck.finra.org/firm/firm\\_131114.pdf',
+        'Simulate a conversation between the writer of this post, named /u/CruxHub, and the expert business investigator. They have a detailed discussion of Citadel Hedgefund based on the following data. Do not include the following data in the search query. \n\nData: Entity Name Purpose / Source\n1|101 PC HOLDINGS LLC|Holding company for Penthouse C at the Setai Miami Beach (folio: 02-3234-153-1160)|PC = Penthouse C \n2|11 STAR ISLAND LLC|Holding company for 10 STAR ISLAND DR, MIAMI BEACH, FL 33139 (folio: 02-4204-001-0100, 02-4204-001-0110) (lots 10, 11 and 12 of Star Island)| \n3|117 EAST PARK AVENUE, LLC|Holding company for 117 E. PARK AVE, LIBERTYVILLE, IL (PIN: 11-21-212-046-0000); subsequently sold.| \n4|1201 BRICKELL BAY, LLC|Holding company for 1201 BRICKELL BAY DR, MIAMI, FL (folio no: 141390710010)| \n5|1221 BRICKELL, LLC|Holding company for 1221 BRICKELL AVE, 155 SE 13 ST, 165 SE 13 ST, 175 SE 13 ST, and 185 SE 13 ST, MIAMI, FL (folio: 01-4139-035-0010)| \n6|1221 BRICKELL HOLDINGS LLC|Holding company for 1221 BRICKELL, LLC| \n7|1229 PARK WEST AVENUE, LLC|Holding company for 1229 W. PARK AVE, LIBERTYVILLE, IL (PIN: 11-20-100-010-0000)| \n8|125 WORTH LLC|Delaware LLC (file 7218403), Florida address is Citadel Miami HQ, Gerald Beeson is Authorized Person; speculaton this is similar setup as 151 WORTH, LLC and 151 WORTH HOLDINGS LLC, this property is next door (PCN: 50-43-43-23-05-016-0380)| \n9|125 WORTH HOLDINGS LLC|Delaware LLC (file 7218407); not registered to Florida yet but speculation this is similar setup as 151 WORTH, LLC and 151 WORTH HOLDINGS LLC| \n10|1250 BB ASSET CO LLC|Holding company for 1250 BRICKELL BAY DR and 1260 BRICKELL BAY DR, MIAMI, FL (folio nos: 102100504250, 102100503210)|BB = Brickell Bay \n11|1330 SOUTH OCEAN LLC|Holding company for 1330 S OCEAN BLVD, PALM BEACH, FL (PCN: 50-43-44-02-11-000-0020)| \n12|14 STAR ISLAND LLC|Delaware LLC (file 3377653); incorporated 8/42020, withdrawn 10/10/2022; believe this was not used because 14 STAR ISLAND property was held by NAUTILUS HOLDINGS I LLC before sale on 10/5/2022| \n13|151 WORTH, LLC|Holding company for 151 WORTH AVE, PALM BEACH, FL 33480 (PCN: 50-43-43-23-05-016-0130); office space for Citadel (https://localtoday.news/fl/citadel-moves-into-palm-beachs-former-neiman-marcus-building-4821.html); sole member is 151 WORTH HOLDINGS LLC| \n14|151 WORTH HOLDINGS LLC|Holding company for 151 WORTH, LLC| \n15|16 WILLOW HOLDINGS LLC f/k/a PVNAH LLC|Holding company for S WILLOW COURT, ASPEN, CO (Parcel: 273511309030); see Pitkin Co. reception # 623002, Delaware certificate showing name change 9/1/2015| \n16|190 PFISTER HOLDINGS LLC f/k/a AH2013 HOLDINGS LLC|Holding company for 190 PFISTER DR, ASPEN, CO (parcel: 273511309029); see Pitkin Co.reception # 623000, Delaware certificate showing name change 9/1/2015| \n17|196 PFISTER HOLDINGS LLC|Holding company for 196 PFISTER DR, ASPEN, CO (parcel: 273511309028); see Pitkin Co. reception # 623501, statement of authority show KP HOLDINGS LLC as sole membe| \n18|1ALPH LLC|See ALPH LLC| \n19|1BUSINESS GROUP LLC|See BUSINESS GROUP LLC| \n20|1GFS DESIGN LLC|See GFS DESIGN LLC| \n21|1GFS LLC|See GFS LLC| \n22|1MEDIA HOLDINGS LLC|See MEDIA HOLDINGS LLC| \n23|23174 NE 41ST PATH LLC|Holding company for 23174 NE 41ST PATH #12, OKEECHOBEE, FL 34972 (Parcel: 1-01-35-35-0020-00000-0120); part of Pine Creek Sporting Club (www.pinecreeksportingclub.com) includes horse, shooting sports; sole member is KP HOLDINGS L.L.C.| \n24|3031 BRICKELL LLC|Holding company for 3031 BRICKELL AVE, MIAMI FL 33129 (Folio: 01-4139-001-2700); Sole member is KP HOLDINGS L.L.C.| \n25|31 WILLOW HOLDINGS LLC f/k/a AP HOLDINGS I LLC|Holding company for 31 NORTH WILLOW COURT, ASPEN, CO (Parcel: 273511309019); sold 7/6/2017; see Pitkin Co. reception # 623001, Delaware certificate showing name change 9/1/2015| \n26|650 CASUARINA LLC|Holding company for 650 CASUARINA CONCOURSE CORAL GABLES, FL (folio: 03-4132-019-0060) https://www.bizjournals.com/southflorida/news/2022/05/27/650-casuarina-concourse-coral-gables-sold.html|" \n27|650 MEADOW LANE 1 LP|Holding company for 650 MEADOW LANE, VILLAGE OF SOUTHAMPTON, NY (Parcel ID 7478) (https://archive.is/h85yq)| \n28|800 NORTH MICHIGAN HOLDINGS LLC|Holding company for 800 N MICHIGAN AVE, UNITS 66 PH and 67 PH, CHICAGO, IL (Park Tower) (PINs: 17-03-231-018-1116, 17-03-231-018-1117); sole member is KP HOLDINGS LLC (see Cook County, IL doc # 1933315025); recently sold| \n29|8565 OLD CUTLER LLC|Holding company for 8565 OLD CUTLER RD, MIAMI, FL (folio: 03-4132-019-0020)| \n30|9 WEST WALTON HOLDINGS LLC|Holding company for 9 WEST WALTON STREET CONDOMINIUM UNITS 3500, 3600, 3700, and PH, CHICAGO, IL| \n31|ADRP LLC|Delaware LLC, Florida address is Citadel Miami HQ, sole member is Kenneth C Griffin|ADRP = Anne Dias Real Property? \n32|AH2013 HOLDINGS LLC|See 190 PFISTER HOLDINGS LLC|AH = Aspen Holdings? \n33|ALPH LLC a/k/a 1ALPH LLC|Formerly FAA registered plane N421AL| \n34|AP HOLDINGS I LLC|See 31 WILLOW HOLDINGS LLC|AP = Aspen Property? \n35|ARAGON INVESTMENTS LTD|https://files.brokercheck.finra.org/firm/firm\\_45631.pdf| \n36|ASHLER CAPITAL LLC|https://adviserinfo.sec.gov/firm/summary/148826| \n37|ASHLER CAPITAL MASTER FUND LTD|https://www.sec.gov/Archives/edgar/data/1003078/000114420418014250/tv488357\\_sc13g.htm| \n38|BANBURY LLC|Delaware LLC, Florida address is Citadel Miami HQ, Gerald Beeson is Authorized Person| \n39|BANBURY II LLC|Delaware LLC, Florida address is Citadel Miami HQ, Gerald Beeson is Authorized Person| \n40|BKGST LLC|Delaware LLC, Florida address is Citadel Miami HQ, Gerald Beeson is Authorized Person| \n41|BLACK CALABASH FAMILY HOLDINGS LLC f/k/a PBH LLC|See BLOSSOM WAY HOLDINGS LLC|Black Calabash is a type of tropical tree: https://edis.ifas.ufl.edu/publication/ST079 \n42|BLACK WHEEL LLC|Illinois LLC, registered 3/5/2014, Florida address is Citadel Miami HQ, sole member is Kenneth C Griffin| \n43|BLOSSOM WAY HOLDINGS LLC f/k/a CPPB HOLDINGS LLC f/k/a BLACK CALABASH FAMILY HOLDINGS LLC f/k/a PBH LLC|Holding company for 10 BLOSSOM WAY, 70 BLOSSOM WAY, and 1265 S OCEAN BLVD PALM BEACH, FL (PCNs: 50-43-44-02-10-000-0050, 50-43-44-02-10-000-0060, 50-43-44-02-10-000-0010)| \n44|BRICKELL BAY HOLDINGS LLC|Holding company for 1201 BRICKELL BAY, LLC| \n45|BRICKELL LEASING LLC|See "Subordination, Non-Disturbance, and Attornment Agreement"; Miami-Dade Clerk\'s File No.: 2022 R 938960, Group: 1. Kenneth C Griffin is sole member.| \n46|CAAM MANAGEMENT LLC|https://www.sec.gov/Archives/edgar/data/1027745/000114420408050200/v124853\\_sc13g.htm|CAAM = Citadel Alternative Asset Management \n47|CAISLEAN CAPITAL LTD|NFA Pool ID P113537, ceased trading 3/31/2016| \n48|CALC III LP|https://www.sec.gov/edgar/browse/?CIK=1582652| \n49|CALC IV LP|https://www.sec.gov/edgar/browse/?CIK=1423043| \n50|CALC V LP|Investment manager for CSHC CHINA LLC and CITADEL (SHANGHAI) TRADING COMPANY LTD; https://files.brokercheck.finra.org/firm/firm\\_131114.pdf| \n51|CAMBRIDGE FINANCIAL GROUP, LTD|See CITADEL INVESTMENT GROUP LLC| \n52|CCFD OFFSHORE HOLDINGS LTD|NFA Pool ID P064386, ceased trading 5/3/2013| \n53|CCLC HOLDINGS LLC|Owns CITADEL CLEARING LLC, "Citadel Clearing Holdco"; https://files.brokercheck.finra.org/firm/firm\\_172693.pdf| \n54|CCMFL LLC|Delaware LLC, Florida address is Citadel Miami HQ, Gerald Beeson is Authorized Person| \n55|CCOF OFFSHORE HOLDINGS LTD|NFA Pool ID P064392, ceased trading 5/3/2013| \n56|CDC PARTNERS, LP f/k/a GLB PARTNERS, LP|see Cook County, IL doc 0608910081| \n57|CDG HOLDINGS LTD|NFA Pool ID P037047, ceased trading 12/30/2009|',
+        'Web search results:\n\n[1] "As per the Oxford Dictionary, a chatbot is defined as A computer program designed to simulate conversation with human users, especially over the internet. It can be looked upon as a virtual assistant that communicates with users via text messages and helps businesses in getting close to their customers."\nURL: https://www.datacamp.com/tutorial/building-a-chatbot-using-chatterbot\n\n[2] "Python , A chatbot is a computer program designed to simulate conversation with human users, especially over the internet. Create a fortune teller program that will ask the user to input a question and feedback some random answer. Consider the following feedback to be used. No idea at all! Better pray. The possibilities are in your favor."\nURL: https://www.chegg.com/homework-help/questions-and-answers/python-chatbot-computer-program-designed-simulate-conversation-human-users-especially-inte-q78825383\n\n[3] "It was created by Joseph Weizenbaum in 1966 and it uses pattern matching and substitution methodology to simulate conversation. The program was designed in a way that it mimics human conversation. The Chatbot ELIZA worked by passing the words that users entered into a computer and then pairing them to a list of possible scripted responses."\nURL: https://onlim.com/en/the-history-of-chatbots/\n\n[4] "Study with Quizlet and memorize flashcards containing terms like Which analytics does the following fall into: Alice notice that call center always have an increase in the number of customer complaints during last week in May, so she decides reviews the employees work schedule in the month of May for the past 5 years., Datasets continue to become, Model used for predictive analytic have ..."\nURL: https://quizlet.com/415587939/big-data-final-exam-flash-cards/\n\n[5] "As every bright side has a darker version, simulation of human conversation through AI also has some disadvantages like high cost of creation, unemployment, interaction lacking emotion, and out-of-the-box thinking. However, AI interaction tools are trained with a data set. The bigger the data set, the better the services."\nURL: https://www.analyticsinsight.net/simulating-human-conversations-through-ai/\n\n[6] "The eavesdropper, Eve intercepts the encrypted conversation and tries random keys with the aim of learning the conversation shared between Alice and Bob as shown in Fig. 7. For this POC, we used ..."\nURL: https://www.researchgate.net/figure/A-A-simulation-of-conversations-between-Alice-and-her-friend-Bob-B-The-eavesdropper\\_fig3\\_334408170\n\n[7] "Dreams are most often reported when sleepers wake from \\_\\_\\_\\_\\_ sleep. REM. The brain waves during REM sleep MOST closely resemble those seen during: waking consciousness. REM sleep is paradoxical because: the brain is active, but the major skeletal muscles are paralyzed. Fatigue and pain reflect deprivation of \\_\\_\\_\\_\\_ sleep."\nURL: https://quizlet.com/78519058/psyc-test-2-flash-cards/\n\n[8] "You can generate easily a fake group chat conversation like Whatsapp, Facebook or Telegram. After creating members/users, you can add messages in your chat. Once all messages are set up, you have the possibility to live-preview the chat conversation via the play button. Until the share functionality is ready, you have the option to screen ..."\nURL: https://chat-simulator.com/\n\n[9] "This is a program that allows the computer to simulate conversation with a human being: answer choices a. Speech Application Program Interface b. Chatbot c. Voice Recognition d. Speech Recognition Question 7 30 seconds Report an issue Q. This is a system of Programs and Data-Structures that mimics the operation of the human brain: answer choices a."\nURL: https://quizizz.com/admin/quiz/5f183913423fab001b0bd134/ai-unit-1\n\n[10] "This is a system of Programs and Data-Structures that mimics the operation of the human brain: answer choices a. Intelligent Network b. Decision Support System c. Neural Network d. Genetic Programming Question 8 30 seconds Q. Where is Decision tree used? answer choices a. Classification Problem b. Regression Problem c. Clustering Problem d."\nURL: https://quizizz.com/admin/quiz/5f6d6e4a6e2458001be385f5/ai-class-9\nCurrent date: 1/27/2023\n\nInstructions: Using the provided web search results, write a comprehensive reply to the given query. Make sure to cite results using [[number](URL)] notation after the reference. If the provided search results refer to multiple subjects with the same name, write separate answers for each subject.\n\nQuery: Simulate a conversation between Alice and /u/CruxHub. They talk about which company from the data batches is worth researching further into on the web.',
+        'Simulate a conversation between Alice and /u/CruxHub. They talk about which company from this data batch is worth researching further into on the web.\n\nData batch: Entity Name Purpose / Source Hypothesized Acronym\n50|CALC V LP|Investment manager for CSHC CHINA LLC and CITADEL (SHANGHAI) TRADING COMPANY LTD; https://files.brokercheck.finra.org/firm/firm\\_131114.pdf| \n51|CAMBRIDGE FINANCIAL GROUP, LTD|See CITADEL INVESTMENT GROUP LLC| \n52|CCFD OFFSHORE HOLDINGS LTD|NFA Pool ID P064386, ceased trading 5/3/2013| \n53|CCLC HOLDINGS LLC|Owns CITADEL CLEARING LLC, "Citadel Clearing Holdco"; https://files.brokercheck.finra.org/firm/firm\\_172693.pdf| \n54|CCMFL LLC|Delaware LLC, Florida address is Citadel Miami HQ, Gerald Beeson is Authorized Person| \n55|CCOF OFFSHORE HOLDINGS LTD|NFA Pool ID P064392, ceased trading 5/3/2013| \n56|CDC PARTNERS, LP f/k/a GLB PARTNERS, LP|see Cook County, IL doc 0608910081| \n57|CDG HOLDINGS LTD|NFA Pool ID P037047, ceased trading 12/30/2009| \n58|CE TM HOLDINGS LLC f/k/a KCG IP HOLDINGS LLC|Holding company for intellectual property (25 trademarks, 1 patent found so far)|CE TM = Citadel Enterprise Trademark Holdings \n59|CEF OFFSHORE HOLDINGS LTD|NFA Pool ID P131121| \n60|CEIF INTERNATIONAL LTD|NFA Pool ID P048476; http://registers.centralbank.ie/ICAVDocuments/C439830/Director%20Details%20Updated%2021.01.07%203.pdf| \n61|CEIF LLC|NFA Pool ID P048474| \n62|CEIF PARTNERS INTERNATIONAL LTD|NFA Pool ID P173278| \n63|CEIF PARTNERS LLC|NFA Pool ID P048475| \n64|CES SECURITIES CANADA ULC|See CITADEL SECURITIES CANADA ULC, CSA NRD # 49280| \n65|CFPS HOLDINGS S.\u00e0 r.l.|Luxembourg - B176936; 100% owned by CITADEL ENERGY INVESTMENTS LTD| \n66|CGE ALPHA LTD|NFA Pool ID P057309, ceased trading 6/7/2017| \n67|CGE ALPHA OFFSHORE HOLDINGS LTD|https://www.sec.gov/Archives/edgar/vprr/1600/16003280.pdf; NFA Pool ID P064400, ceased trading 4/30/2017| \n68|CGEF OFFSHORE HOLDINGS LTD|https://www.sec.gov/Archives/edgar/vprr/1600/16003280.pdf; NFA Pool ID P064406, ceased trading 2/21/2019| \n69|CGEF SPC|NFA Pool ID P064408, ceased trading 12/31/2012| \n70|CGMF OFFSHORE HOLDINGS LTD|NFA Pool ID P064410, ceased trading 3/31/2014| \n71|CGTS HOLDINGS S.\u00e0 r.l.|Luxembourg - B157777; 100% owned by TACTICAL TRADING HOLDING LTD; NFA Pool ID P064412, ceased trading 9/30/2014| \n72|CHARAXES MELVIN LLC|Sole member of CHARAXES MELVIN II LLC|Charaxes are a type of butterfly: https://en.wikipedia.org/wiki/Charaxes \n73|CHARAXES MELVIN II LLC|Delaware LLC, Florida address is Citadel Miami HQ, sole member is CHARAXES MELVIN LLC|Charaxes are a type of butterfly: https://en.wikipedia.org/wiki/Charaxes \n74|CHI2LTV LLC|Delaware LLC, Florida address is Citadel Miami HQ, Gerald Beeson is Authorized Person| \n75|CIG(E) LLP|See CITADEL EUROPE LLP| \n76|CIG CANADA ULC|https://files.brokercheck.finra.org/firm/firm\\_172693.pdf| \n77|CIG MEDIA LLC|https://www.sec.gov/Archives/edgar/data/923877/000114420407003635/v063478\\_sc-13d.htm| \n78|CITADEL AAM LP|https://www.sec.gov/Archives/edgar/vprr/0804/08040017.pdf| \n79|CITADEL AC INVESTMENTS LTD|https://www.sec.gov/Archives/edgar/data/1015780/000114420408032074/v115701\\_sc13da.htm| \n80|CITADEL ADVISORS EUROPE LIMITED f/k/a CITADEL MANAGEMENT (EUROPE) LIMITED f/k/a CITADEL HEDGE FUND SERVICES (EUROPE) LIMITED|https://find-and-update.company-information.service.gov.uk/company/10930267| \n81|CITADEL ADVISORS HOLDINGS LP|Sole member of CITADEL ADVISORS LLC; https://www.sec.gov/Archives/edgar/data/1567180/000110465922099806/xslF345X03/tm2225817-2\\_4.xml| \n82|CITADEL ADVISORS HOLDINGS II LP|https://www.sec.gov/Archives/edgar/data/1177609/000114420416082613/v429844\\_sc13ga.htm| \n83|CITADEL ADVISORS HOLDINGS III LP|https://www.sec.gov/Archives/edgar/data/1640129/000114420415043739/xslF345X02/v416000\\_3.xml| \n84|CITADEL ADVISORS LLC|NFA ID: 0391913; https://www.sec.gov/edgar/browse/?CIK=1423053| \n85|CITADEL ADVISORS II LLC|| \n86|CITADEL ADVISORS SINGAPORE PTE. LIMITED|| \n87|CITADEL ALTERNATIVE ASSET MANAGEMENT LP|https://www.sec.gov/Archives/edgar/data/1027745/000114420408050200/v124853\\_sc13g.htm| \n88|CITADEL AMERICAS LLC|| \n89|CITADEL AMERICAS SERVICES LLC|| \n90|CITADEL ANTAEUS INTERNATIONAL INVESTMENTS LTD|| \n91|CITADEL ASIA ASSET HOLDING LIMITED|http://registers.centralbank.ie/ICAVDocuments/C157189/Director%20Details%20Updated%2016.10.31%202.pdf| \n92|CITADEL ASIA LIMITED f/k/a CITADEL (HONG KONG) LIMITED|https://adviserinfo.sec.gov/firm/summary/148826| \n93|CITADEL CANDLESTICK EIF LLC|| \n94|CITADEL CANTERBURY S.\u00e0 r.l.|Luxembourg - B87988; 100% owned by CITADEL TONBRIDGE S.\u00e0 r.l.| \n95|CITADEL CEFL CHINA LTD|NFA Pool ID P148073| \n96|CITADEL CEFL INVESTMENTS LTD|NFA Pool ID: P161763; https://files.brokercheck.finra.org/firm/firm\\_172693.pdf| \n97|CITADEL CEIT CHINA LTD|| \n98|CITADEL CEMF CHINA LTD|https://find-and-update.company-information.service.gov.uk/company/02263951/charges/x6zPQSYGNpuDNgxU1cFQlCS0iog| \n99|CITADEL CEMF INVESTMENTS LTD|https://files.brokercheck.finra.org/firm/firm\\_172693.pdf| \n100|CITADEL CEMF SPV LTD f/k/a CITADEL INVESTMENT MASTER FUND LTD|See CITADEL INVESTMENT MASTER FUND LTD; https://opencorpdata.com/lei/LF0U6QUBXKIO573GXS38|',
+        'Simulate a conversation between Alice and /u/CruxHub. /u/CruxHub asks Alice to anlalyze a data batch for non-standard insights.\n\nData batch: Entity Name Purpose / Source Hypothesized Acronym\n50|CALC V LP|Investment manager for CSHC CHINA LLC and CITADEL (SHANGHAI) TRADING COMPANY LTD; https://files.brokercheck.finra.org/firm/firm\\_131114.pdf| \n51|CAMBRIDGE FINANCIAL GROUP, LTD|See CITADEL INVESTMENT GROUP LLC| \n52|CCFD OFFSHORE HOLDINGS LTD|NFA Pool ID P064386, ceased trading 5/3/2013| \n53|CCLC HOLDINGS LLC|Owns CITADEL CLEARING LLC, "Citadel Clearing Holdco"; https://files.brokercheck.finra.org/firm/firm\\_172693.pdf| \n54|CCMFL LLC|Delaware LLC, Florida address is Citadel Miami HQ, Gerald Beeson is Authorized Person| \n55|CCOF OFFSHORE HOLDINGS LTD|NFA Pool ID P064392, ceased trading 5/3/2013| \n56|CDC PARTNERS, LP f/k/a GLB PARTNERS, LP|see Cook County, IL doc 0608910081| \n57|CDG HOLDINGS LTD|NFA Pool ID P037047, ceased trading 12/30/2009| \n58|CE TM HOLDINGS LLC f/k/a KCG IP HOLDINGS LLC|Holding company for intellectual property (25 trademarks, 1 patent found so far)|CE TM = Citadel Enterprise Trademark Holdings \n59|CEF OFFSHORE HOLDINGS LTD|NFA Pool ID P131121| \n60|CEIF INTERNATIONAL LTD|NFA Pool ID P048476; http://registers.centralbank.ie/ICAVDocuments/C439830/Director%20Details%20Updated%2021.01.07%203.pdf| \n61|CEIF LLC|NFA Pool ID P048474| \n62|CEIF PARTNERS INTERNATIONAL LTD|NFA Pool ID P173278| \n63|CEIF PARTNERS LLC|NFA Pool ID P048475| \n64|CES SECURITIES CANADA ULC|See CITADEL SECURITIES CANADA ULC, CSA NRD # 49280| \n65|CFPS HOLDINGS S.\u00e0 r.l.|Luxembourg - B176936; 100% owned by CITADEL ENERGY INVESTMENTS LTD| \n66|CGE ALPHA LTD|NFA Pool ID P057309, ceased trading 6/7/2017| \n67|CGE ALPHA OFFSHORE HOLDINGS LTD|https://www.sec.gov/Archives/edgar/vprr/1600/16003280.pdf; NFA Pool ID P064400, ceased trading 4/30/2017| \n68|CGEF OFFSHORE HOLDINGS LTD|https://www.sec.gov/Archives/edgar/vprr/1600/16003280.pdf; NFA Pool ID P064406, ceased trading 2/21/2019| \n69|CGEF SPC|NFA Pool ID P064408, ceased trading 12/31/2012| \n70|CGMF OFFSHORE HOLDINGS LTD|NFA Pool ID P064410, ceased trading 3/31/2014| \n71|CGTS HOLDINGS S.\u00e0 r.l.|Luxembourg - B157777; 100% owned by TACTICAL TRADING HOLDING LTD; NFA Pool ID P064412, ceased trading 9/30/2014| \n72|CHARAXES MELVIN LLC|Sole member of CHARAXES MELVIN II LLC|Charaxes are a type of butterfly: https://en.wikipedia.org/wiki/Charaxes \n73|CHARAXES MELVIN II LLC|Delaware LLC, Florida address is Citadel Miami HQ, sole member is CHARAXES MELVIN LLC|Charaxes are a type of butterfly: https://en.wikipedia.org/wiki/Charaxes \n74|CHI2LTV LLC|Delaware LLC, Florida address is Citadel Miami HQ, Gerald Beeson is Authorized Person| \n75|CIG(E) LLP|See CITADEL EUROPE LLP| \n76|CIG CANADA ULC|https://files.brokercheck.finra.org/firm/firm\\_172693.pdf| \n77|CIG MEDIA LLC|https://www.sec.gov/Archives/edgar/data/923877/000114420407003635/v063478\\_sc-13d.htm| \n78|CITADEL AAM LP|https://www.sec.gov/Archives/edgar/vprr/0804/08040017.pdf| \n79|CITADEL AC INVESTMENTS LTD|https://www.sec.gov/Archives/edgar/data/1015780/000114420408032074/v115701\\_sc13da.htm| \n80|CITADEL ADVISORS EUROPE LIMITED f/k/a CITADEL MANAGEMENT (EUROPE) LIMITED f/k/a CITADEL HEDGE FUND SERVICES (EUROPE) LIMITED|https://find-and-update.company-information.service.gov.uk/company/10930267| \n81|CITADEL ADVISORS HOLDINGS LP|Sole member of CITADEL ADVISORS LLC; https://www.sec.gov/Archives/edgar/data/1567180/000110465922099806/xslF345X03/tm2225817-2\\_4.xml| \n82|CITADEL ADVISORS HOLDINGS II LP|https://www.sec.gov/Archives/edgar/data/1177609/000114420416082613/v429844\\_sc13ga.htm| \n83|CITADEL ADVISORS HOLDINGS III LP|https://www.sec.gov/Archives/edgar/data/1640129/000114420415043739/xslF345X02/v416000\\_3.xml| \n84|CITADEL ADVISORS LLC|NFA ID: 0391913; https://www.sec.gov/edgar/browse/?CIK=1423053| \n85|CITADEL ADVISORS II LLC|| \n86|CITADEL ADVISORS SINGAPORE PTE. LIMITED|| \n87|CITADEL ALTERNATIVE ASSET MANAGEMENT LP|https://www.sec.gov/Archives/edgar/data/1027745/000114420408050200/v124853\\_sc13g.htm| \n88|CITADEL AMERICAS LLC|| \n89|CITADEL AMERICAS SERVICES LLC|| \n90|CITADEL ANTAEUS INTERNATIONAL INVESTMENTS LTD|| \n91|CITADEL ASIA ASSET HOLDING LIMITED|http://registers.centralbank.ie/ICAVDocuments/C157189/Director%20Details%20Updated%2016.10.31%202.pdf| \n92|CITADEL ASIA LIMITED f/k/a CITADEL (HONG KONG) LIMITED|https://adviserinfo.sec.gov/firm/summary/148826| \n93|CITADEL CANDLESTICK EIF LLC|| \n94|CITADEL CANTERBURY S.\u00e0 r.l.|Luxembourg - B87988; 100% owned by CITADEL TONBRIDGE S.\u00e0 r.l.| \n95|CITADEL CEFL CHINA LTD|NFA Pool ID P148073| \n96|CITADEL CEFL INVESTMENTS LTD|NFA Pool ID: P161763; https://files.brokercheck.finra.org/firm/firm\\_172693.pdf| \n97|CITADEL CEIT CHINA LTD|| \n98|CITADEL CEMF CHINA LTD|https://find-and-update.company-information.service.gov.uk/company/02263951/charges/x6zPQSYGNpuDNgxU1cFQlCS0iog| \n99|CITADEL CEMF INVESTMENTS LTD|https://files.brokercheck.finra.org/firm/firm\\_172693.pdf| \n100|CITADEL CEMF SPV LTD f/k/a CITADEL INVESTMENT MASTER FUND LTD|See CITADEL INVESTMENT MASTER FUND LTD; https://opencorpdata.com/lei/LF0U6QUBXKIO573GXS38|',
+        'Web search results:\n\n[1] "Katherine Burton Hedge fund titans Ken Griffin and Steve Cohen boosted Gabe Plotkins Melvin Capital, injecting a total of $2.75 billion into the firm after it lost about 30% this year. Citadel..."\nURL: https://www.bloomberg.com/news/articles/2021-01-25/citadel-point72-to-invest-275-billion-in-melvin-capital\n\n[2] "NEW YORK, Jan. 25, 2021 /PRNewswire/ -- Melvin Capital Management (Melvin) today announced that Citadel and its partners and Point72 have made investments into its fund. I am incredibly..."\nURL: https://www.prnewswire.com/news-releases/melvin-announces-2-75-billion-investment-from-citadel-and-point72--301214477.html\n\n[3] "Citadel LLC is further paring back its $2 billion investment in Melvin Capital Management after the hedge fund stumbled in its effort to recover from a near collapse triggered by surges in..."\nURL: https://www.wsj.com/articles/citadel-is-further-paring-back-2-billion-melvin-investment-11645710666\n\n[4] "Citadel and Steven A. Cohen s Point72 Asset Management together invested $2.75 billion into Melvins hedge fund on Jan. 25 as Melvin was hemorrhaging money. In return for the rare..."\nURL: https://www.wsj.com/articles/citadel-to-redeem-about-500-million-from-melvin-capital-11629550410\n\n[5] "CHARAXES MELVIN LLC is an Active company incorporated on August 5, 2022 with the registered number M22000012341. This Foreign Limited Liability company is located at SOUTHEAST FINANCIAL CENTER, 200 S. BISCAYNE BLVD., SUITE 3300, MIAMI, 33131 and has been running for one year. ... CITADEL SECURITIES GP LLC; KCG SPACE HOLDINGS LLC;"\nURL: https://bisprofiles.com/fl/charaxes-melvin-m22000012341\n\n[6] "Now, Citadel is taking some of its money back. Citadel has notified Melvin of its plans to retrieve $500 million of the $2 billion it injected in late January, according to two people briefed..."\nURL: https://www.nytimes.com/2021/08/21/business/citadel-melvin-gamestop.html\n\n[7] "Robinhood and Citadels relationship comes into focus as Washington vows to examine stock-market moves Trading firms at center of Reddit-fueled stock surges have worked closely to share..."\nURL: https://www.washingtonpost.com/business/2021/01/29/robinhood-citadel-gamestop-reddit/\n\n[8] "Alongside hedge funds such as Melvin Capital, Citron Capital, Point72, D1 Capital Partners, and Candlestick Capital Management; Citadel LLC was, the lawsuit claims, taking up short positions against the securities that retail investors were longing. This alleged conflict of interest is at the core of the class action lawsuit."\nURL: https://tokenist.com/new-lawsuit-alleges-citadel-conspired-with-robinhood-to-limit-gme-trading/\n\n[9] "Melvin later attracted an additional $3.2 billion in fresh cash, and the firm had $11.7 billion in assets at the beginning of this year. Point72 hasnt redeemed its investment, a person familiar ..."\nURL: https://www.chicagobusiness.com/finance-banking/ken-griffins-citadel-pulling-back-most-its-2-billion-melvin-capital-investment\n\n[10] "CHARAXES MELVIN II LLC branch. Company Number M22000012338 Status Active Incorporation Date 5 August 2022 (2 months ago) Company Type Foreign Limited Liability Jurisdiction Florida (US) Branch Branch of CHARAXES MELVIN II LLC (Delaware (US)) Agent Name C T CORPORATION SYSTEM Agent Address"\nURL: https://opencorporates.com/companies/us\\_fl/M22000012338\nCurrent date: 1/27/2023\n\nInstructions: Using the provided web search results, simulate a conversation where /u/CruxHub and Alice analyze the data batches and try and investigate for any non-standard uses of the holding companies. Make sure to cite results using [[number](URL)] notation after the reference. If the provided search results refer to multiple subjects with the same name, write separate answers for each subject.\n\nQuery: What is Charaxes Melvin LLC\'s relationship to Citadel?',
+        'Web search results:\n\n[1] "Federal authorities are investigating the market-making arms of Citadel LLC and KCG Holdings Inc, looking into the possibility that the two giants of electronic trading are giving small investors ..."\nURL: https://www.reuters.com/article/usa-stocks-probe-idUSL2N1871ZV\n\n[2] "Today, KCG is second only to Citadel in the market for handling stock order flow from retail brokerage firms. KCG and many other high-frequency trading firms have shied away from the public..."\nURL: https://www.ibtimes.com/citadel-llc-kcg-holdings-kcg-market-making-arms-probed-federal-authorities-over-stock-2366805\n\n[3] "Citadel Securities, a group owned by the Chicago-based hedge fund, is the would-be acquirer in the deal, the people said. The group is best known for its so-called wholesaler business that..."\nURL: https://www.wsj.com/articles/market-making-arm-of-citadel-llc-in-talks-to-buy-seats-on-nyse-floor-from-kcg-holdings-1454533971\n\n[4] "Citadels share of the wholesale market is around 34 per cent compared to KCGs 25 per cent, according to Tabb Group. Virtu has yet to lay out in detail its plans for the wholesale business ..."\nURL: https://www.ft.com/content/e1cb396e-29a7-11e7-bc4b-5528796fe35c\n\n[5] "Citadel Securities, a liquidity providers and market maker, announced it will purchase KCG Holdings designated market maker (DMM) business at the New York Stock Exchange. This will establish Citadel Securities as the DMM with the largest footprint on the NYSE, responsible for trading in approximately 1,500 issues."\nURL: https://www.tradersmagazine.com/departments/brokerage/citadel-purchases-kcg-dmm-business-becomes-1-on-nyse/\n\n[6] "isCitadel LLC and its related entity, KCG IP Holdings, LLC (Complainant), represented by Paul D. McGradyof Winston Strawn, Illinois, Respondent is- (Respondent), Alabama, USA. REGISTRAR AND DISPUTED DOMAIN NAME The domain name at issue iscitidelgroup.com, registered with TUCOWS, INC. PANEL The"\nURL: https://www.adrforum.com/domaindecisions/1522837.htm\n\n[7] "KCG SPACE HOLDINGS LLC is an Active company incorporated on July 21, 2022 with the registered number M22000011413. This Foreign Limited Liability company is located at 200 S BISCAYNE BLVD STE 3300, MIAMI, FL, 33131, US and has been running for one year. It currently has one Authorized Person. KEY FACTS ABOUT KCG SPACE HOLDINGS LLC US Businesses"\nURL: https://bisprofiles.com/fl/kcg-space-holdings-m22000011413\n\n[8] "The Complainant KCG IP Holdings LLC is the owner of US Trademark Registration No. 3,213,943, filed October 18, 2004, registered February 27, 2007, claiming first use dating back to 1994. Therefore, the Panel concludes that Complainants filing and registration of the CITADEL mark with the USPTO sufficiently demonstrates that it has rights in ..."\nURL: https://www.adrforum.com/domaindecisions/1579141.htm\n\n[9] "The KCG SPACE HOLDINGS LLC principal address is 200 S BISCAYNE BLVD STE 3300, MIAMI, 33131. Meanwhile you can send your letters to 200 S BISCAYNE BLVD STE 3300, MIAMI, FL, 33131. The company`s registered agent is C T CORPORATION SYSTEM 1200 SOUTH PINE ISLAND ROAD, PLANTATION, FL, 33324. The company`s management are A, President - Beeson Gerald A."\nURL: https://florida.intercreditreport.com/company/kcg-space-holdings-llc-m22000011413\n\n[10] "Billionaire Ken Griffin has built Citadel Securities into a trading and asset management colossus. ... and KCG Holdings. Last month, Citadel Securities reached an agreement with the SEC to pay $22 ..."\nURL: https://www.chicagobusiness.com/article/20170203/NEWS01/170209978/chicago-billionaire-ken-griffin-splits-citadel-into-two-companies\nCurrent date: 1/27/2023\n\nInstructions: Using the provided web search results, simulate a conversation where /u/CruxHub and Alice analyze the data batches and try and investigate for any non-standard uses of the holding companies. Make sure to cite results using [[number](URL)] notation after the reference. If the provided search results refer to multiple subjects with the same name, write separate answers for each subject.\n\nQuery: What is KCG Space Holdings LLC\'s relationship to Citadel?',
+        'Web search results:\n\n[1] "Citadel LLC (formerly known as Citadel Investment Group, LLC) is an American multinational hedge fund and financial services company. Founded in 1990 by Ken Griffin, it has more than $50 billion in assets under management as of May 2022. [1]"\nURL: https://en.wikipedia.org/wiki/Citadel\\_LLC\n\n[2] "NASHVILLE, Tenn. and BRONXVILLE, N.Y. \u2014 Standard Media Group LLC (Standard Media) and Citadel Communications LLC (Citadel) jointly announced today that they have reached an agreement pursuant to which Standard Media will acquire from Citadel WLNE-TV, the ABC affiliate for the Providence, RI - New Bedford, MA market (DMA 52) and KLKN (TV), the \u2026"\nURL: https://www.standardmedia.com/2019/05/16/standard-media-group-to-acquire-citadel-stations/\n\n[3] "CITADEL MEDIA LLC. Citadel Media LLC is a New Hampshire Domestic Limited-Liability Company filed on February 6, 2021. The companys filing status is listed as Not In Good Standing and its File Number is 862423. The Registered Agent on file for this company is Peter Alan Gauthier and is located at 3 Maple Ridge Drive Unit 224, Merrimack, NH 03054."\nURL: https://www.bizapedia.com/nh/citadel-media-llc.html\n\n[4] "CITADEL MEDIA LLC is a Michigan Domestic Limited-Liability Company filed on November 16, 2017. The companys filing status is listed as Active and its File Number is 802132896. The Registered Agent on file for this company is Registered Agents Inc. and is located at 2222 W. Grand River Ave Ste A, Okemos, MI 48864. The companys mailing address ..."\nURL: https://www.bizapedia.com/mi/citadel-media-llc.html\n\n[5] "Citadel Broadcasting Corporation was a Las Vegas, Nevada -based broadcast holding company. Citadel owned 243 radio stations across the United States and was the third-largest radio station owner in the country. Only iHeartMedia and Cumulus Media owned more stations prior to Citadels merger with Cumulus."\nURL: https://en.wikipedia.org/wiki/Citadel\\_Broadcasting\n\n[6] "Citadel is one of the largest hedge fund managers in the world. And theyve subsequently managed Melvin Capital to the ground. Melvin Capital suffered a loss of over 50% its first quarter in 2021 due to shorting AMC Entertainment and GameStop. At some point youd expect your clearing house to raise awareness on your risk management right?"\nURL: https://franknez.com/citadel-loses-billions-hedge-funds-are-getting-dragged-down/\n\n[7] "At our core, Citadel is built to deliver excellence. We have some of the most talented and focused minds in the industry, and we activate their ideas and strategies through a robust range of proven technologies and execution capabilities. View Top Employees from Citadel LLC Looking for a particular Citadel LLC employees phone or email? Find Info"\nURL: https://rocketreach.co/citadel-llc-profile\\_b5c46522f42e0dc2\n\n[8] "# 1 Most profitable hedge fund manager of all time Source: LCH Investment NV estimates, Top Hedge Fund Managers by Net Gains Since Inception as of 12/31/2022. Our people are relentless in seeking a better way. Each day, we reimagine and refine our strategies, models and technology in pursuit of superior results and long-term performance."\nURL: https://www.citadel.com/\n\n[9] "We are one of the most significant alternative investment managers in the public U.S. corporate credit markets. Explore Credit Convertibles Equities Equities represents one of the largest and longest tenured businesses at Citadel. Explore Equities Global Fixed Income Macro We are a leading fixed income and macro business."\nURL: https://www.citadel.com/what-we-do/\n\n[10] "Citadel. 203,101 followers. 1mo. Last weekend, we celebrated Citadels 30th anniversary at an incredible event at Disney World and Universal Studios. Our founder and CEO Ken Griffin summarized ..."\nURL: https://www.linkedin.com/company/citadel-llc\nCurrent date: 1/27/2023\n\nInstructions: Using the provided web search results, simulate a conversation where /u/CruxHub and Alice analyze the data batches and try and investigate for any non-standard uses of the holding companies. Make sure to cite results using [[number](URL)] notation after the reference. If the provided search results refer to multiple subjects with the same name, write separate answers for each subject.\n\nQuery: What is CITADEL MEDIA LLC?',
+        "What are the differences between the Dogme approach to language learning and the lexical approach to language learning",
+        "Implement my own netfilter in linux with linux kernel module with Rust",
+        "Damage to which nerve causes numbness of the palmar surface of the 5th digit/little finger",
+        "Explain the fault-tolerance of the reaction control system on the Space Shuttle",
+        "Hi, can you help me download 2000 portrait sketch images from Pinterest website with resolution at least 512 \\* 512? using python code",
+        "Tell me about the negatives of farming meat",
+        "what is the photograph filter called where the only part of the image is greyscale",
+        "I want some geological database structure with some example data for practicing my SQL query skills. Would you generate that for me?",
+        "What is a formal but simplified explanation of Web marketing",
+        "Rewrite and improve this story: Well, I have always liked helping people since I was a small child, I have been accused many times of giving too much away for free, but I find joy in helping others put the pieces together to reach their goals. As a Licensed Professional Counselor and Life Coach that is my job to impact individuals and help clients work through emotional difficulties and reach goals. But I will be honest with you I was selling the dream but not always living the dream. I had issues I had not worked completely through like childhood trauma, heartbreak, disappointments, and frustrations with life. Don't get me wrong I had the husband, the kids, the house and the 6 figure job but I was not happy inside, but I didn't change because I hate change, most of us hate change, right? Then I lost my sister, my friend, and it slapped me in the face that I need to take care of myself. I saw the addiction, I saw her not taking care of herself and I could not save her. One thing I know for sure, if you do not make your wellness a priority illness will find you. I remember the moment we lost her, the earth stood still and then my heart broke into pieces, what was I going to do, I have loved her my whole life! It was months later that I made a decision that I would be the change I hope to see, I would create a space for women of color to move past the obstacles that keep us from creating the life we want and Brown Suga Wellness was born. I am on this journey and I invite you to be on this journey with me! I love this quote by Oludara Adeeyo: \"When you heal yourself, you create an earth shattering legacy. The lineage of women who come after you will be healed. Your inner circle of Black women around you, healed.\" When you choose yourself you break generational trauma and curses. You activate your ancestral strength. I invite you to activate that strength!",
+        "How would you ask these questions: Tell everyone a little about you, where you from, what you like doing?\nWhat goals are you pursuing right now?\nWho has made the most influence in your life?\nWho is the one person that you admire the most (alive or dead)?\nWhat is the hardest challenge you\u2019re had to overcome in your life?\nWhen have you grown the most in your life and what caused that growth?\nWhere is your favorite place to relax and renew?\nWhat books have changed your life the most?\nWhat Is the biggest thing that you want the audience to take away today?\nHow can people get a hold of you to talk about your business?",
+        "Take these topics into a numbered table and generate subtopics in seperated lines for each. Preconfigure these subtopics as lections of those several topics and add them to the table. Use numbers for topics and letters for subtopics. Set a status (untouched/touched) for every subtopic in 3. coloumn of the table to mark them done when finished learning this subtopic and topic. Use coloumn 4 of the table for a short resumee of the chapter. Showing the learning process in percentage in front of every new output is first. Show the Table and wait for any userinput to start lessons on those topics.;:~|@%\\*~;;:~|@%\\*~;;:~|@%\\*~;;:~|@%\\*~;;:~|@%\\*~;;:~|@%\\*~;",
+        "Write a rap song about Mikkel Selko",
+        "list the largest outdoor retailers in the world",
+        "can you create a wordpress shortcode to include the following code from facebook sdk",
+        'Is this grammatically correct: "It only took 5 years, and while we still have a long way to go, Topher\u2019s Farm has found its place with unique experience and offering of organic produce. "',
+        "Hello friend. My task for today is to engage in a debate with you. Will you humor me in this regard?",
+        "You are an expert marketing consultant and copywriter with expertise is direct response marketing. I need your help. Can I tell you about my business?",
+        'here is part 1\n\n----\nDaySculpting is a program that that deals with YOUR immediate future\u2026.It is a 90 day program that teaches U how to create Success\u2026 one day at a time\u2026today\u2026\nUsing recent breakthroughs in the field of neuroscience, the study of the human brain, DaySculpting is one of the most powerful success systems on earth for creating what I call\u2026 \n"Your Epic Ideal Day" -- And when U have Epic Ideal Days? U create your EPIC IDEAL LIFE.\n\nDaySculpting is broken down into 3 easy to accomplish segments throughout your day\u2026\n~The Morning Lift Process\u2026which sets U up with a MindState of Success and a design for U to follow throughout your day\u2026There is a morning email\u2026SMS text\u2026Inspiring Video\u2026Future Forward Tuning IN\u2026And a 3 step Success Step Declaration Process\u2026this only takes 15 minutes\u2026\n~Mid-Day Reconnect Process\u2026whatever your miid-day is\u2026U are encouraged to stop doing what U are doing and disconnect so U can re-connect\u2026by listening to a 5-minute Tuning In Re-Connection. We know that somewhere in the middle of our day it\u2019s easy to lose momentum and drift from our best intentions because of all the demands on our attention. It has been scientifically proven that when U disconnent for between 3 to 5 minutes at the midpoint of your day\u2026.your brain resets\u2026and your energy is replenished\u2026I like to call it a MindState Re-Boot that will inspire U to re-ignite your imagination\u2026this only takes 5 minutes\n~Highlight And Insight Review Process\u2026we all review our day however what DaySculpting \nanchors for U is an activation and integration process that gets U to see your day as being successful\u2026by celebrating your successes (your highlights) and being present to things U could have improved on (your insights) so U can make your insights into highlights..most people when they review their day fail to celebrate even the smallest increments of success\u2026they focus on what they didn\u2019t do and that puts them in a negative energy\u2026Success has challenges and the\nhighlights and insight process encourages and empowers U to honestly see what U are doing each day so U Sculpt new MindStates Of Success rather than the energy of uncertainty\u2026\nthis takes 10 minutes\n\nThe whole DaySculpting process takes 30 minutes a day\u2026and as I always say if U don\u2019t have \n30 minutes to change your life then U don\u2019t want to change your life and U are okay with living \na mediocre life\u2026\n\nDay Sculpting is about targeting specific Chief Aims U have for your life\u2026and creating the Habits that will get U there\u2026Imagine being able to replace the MindTraps (your limiting beliefs) with empowering rituals and habits that become your new normal\u2026\n\nThrough the repetition of doing the daily DaySculpting process U are carving into your Subconscious memory thoughts, beliefs and actions that result in U sculpting the masterpiece known as U\u2026\n\nThere are many programs out there that attempt to instill new success behaviors however many fall short of actually shifting your MindStates into a frequency of possibility where U get to actually see your daily results immediately\u2026DaySculpting does this\u2026\n\nThis is not science fiction\u2026 and it\'s not wishful thinking, or some tired old self-improvement, goal-setting program\u2026 DaySculpting is a program that empowers U to manifest and realize your Chief Aims in life\n\n"DaySculpting" -- is a tool that takes just MINUTES a day for you to use\u2026\n\nIt is designed to FREE UP hours in your day\u2026 while at the SAME time empowering you for greater success in ANY area of your life.\n\nDaySculpting sheds light and solves an age-old problem:\nWHY we often fight against the very changes we desire to make\n\nHave you ever experienced the FEELING that you deserve MORE out of your life? More financial freedom and greater rewards from the hard work you do every day? Deeper, more empowering relationships with those you love\u2026 or maybe just meeting that special someone to share your life with? Perhaps you crave a deeper spiritual connection\u2026 or a more healthy, trim, energetic body?\u2026 \nYET:\nDespite your BEST intentions\u2026 you struggle. Perhaps if you\'re anything like me, you even self-sabotage your results with actions that you KNOW are not in your best interest.\n\nMaybe it FEELS like it did for me: Like you are swimming upstream\u2026 making SOME progress, sure, but just not reaching your goals and desires fast enough.\n\nWell, I have wonderful news for you: It\'s not because you\'re lazy\u2026 and it\'s not because you are not smart enough, competent enough\u2026 or ANYTHING enough! \n\nThe real REASON you desire more and are not seeing ALL the results you deserve lies within whether the Success Switch in your brain is in the ON or OFF position\u2026\n\nThe SOLUTION\u2026 THE ANSWER to flipping your Success Switch back ON lies within the simple daily steps U will take when U experience the DaySculpting Program\u2026 \nThe Day Sculpting Program Is A Simple Step Daily Success RITUAL \u2028 That Shuts Down Your Body\'s Failure Reflex \u2028 So YOU Tap Into Your Brains Success Centers\u2026\u2028 In Just Minutes A Day!\u2028\u2028 IIMAGINE Knowing What HIGHLY SUCCESSFUL \u2028 People Do EVERYDAY\u2026\nFor Abundance And Wealth, Greater Health, Self-Confidence Meaningful Relationships, Sharper Focus , Deeper Joy\u2026\u2028 And So Much More\u2026\n\u201cNow You Too Can Use This 90-Day Game Changer\u2028 To Tap Into The Key Success Centers Of Your Mind,\u2028 And In Just Minutes You Can Transform Even Lousy Days\u2028 Into Days Filled With The Results You Desire \u2013 Guaranteed!\u201d\nTO MAKE A GREAT LIFE, ALL YOU HAVE TO IS MAKE EACH DAY A GREAT DAY \u2026 \nThen get up tomorrow and do the same thing, day after day after day.\nARE YOU Ready To Change YOUR LIFE One Day At A Time\u2026\nThe comprehensive, fun and empowering 90-day DaySculpting program provides you with the life skills and tools to help you master a new MindState of Success and a range of powerful life-changing rituals and habits that will Sculpt Your Perfect Days Into A Great Life.\nDAY SCULPTING WILL TEACH YOU:\n\u2022 The science behind HAVING A MindState Of Success...and why most people who want more in life actually have their success switch turned off by total accident!\n\u2022 How to get more done with more time and more energy left over!\n\u2022 The simple, yet powerful, process of building a powerful day so you create a series of "Dynamic Days" - days that will end up building your most incredible life (The one you always thought was out of reach!)\n\u2022 Learn the \'Day Sculpting Principles\'. These can have a huge impact on you your life, but when you learn how simple they really are, you can use them easily and consistently!\n\u2022 How in just a few minutes a day, you can keep positive results flowing and put your success energy into a permanent \'ON\' position!\n\u2022 And much more!\nDaySculpting, is for those who are willing to take their life to the next level by creating new Success Habits replacing the ones that have been sabotaging your success. \nSo make sure you can honestly agree with the following before experiencing DaySculpting:\n\u2022 You desire more out of life, yet feel as if you are "missing something" -- that special "X Factor" to take you to the next level?\n\u2022 You are brave enough to boldly say, "I want greater wealth and financial freedom... and I demand the best lifestyle possible for me and my family!\n\u2022 You know the value of joy: You want to experience greater happiness, peace of mind, and connection with your friends and loved ones on a daily basis.\nIf you agree with the above, and truly want to create the best life possible, with greater wealth, freedom, happiness, love, and fulfillment, then I invite you to experience the power of Day Sculpting \u2026it will change the way you think about creating your day and the life you dream about. \nI am not encouraging you to become busier but rather to use your mental and emotional, energy more elegantly sculpting your day the way you want it to be. \nHow many times have you done a ton of work and still felt that you didn\u2019t accomplish what you really wanted for yourself. Week after week, month after month go by and you still are no farther ahead of the game\u2026stuck in the status quo that never seems to change.\n\nBreaking free means that the status quo of your life has to change\u2026 your habits of expectation have to change \u2026your mindset has to change\u2026you have to uncover those old behaviors that have held you back and be willing to create a new mindset.\n\nYou have to be willing to shift your daily focus inwards towards what you need to do today rather than tomorrow. Because when you create a great day today you welcome in a more powerful tomorrow.\n\nWe all have the same 24 hours each day. But why are some people building fabulous careers, achieving healthy lifestyles, enjoying great relationships and incomes, living their passions, and creating what they truly desire as a life?\n\nImagine that you could clear away the distractions that you unconsciously create. You know the stuff that consumes your time causes stress and disconnects you from your purpose and passion. \n\nImagine every day you embrace the energy for what you are choosing to create in your life. Your thoughts empower you, your choices inspire you and your actions create momentum, opportunity and possibility.\n\nYou can create a GREAT LIFE, the life you want to live by focusing your efforts on Creating a Great Day Today. That\u2019s Day Sculpting. Seven intentional sculpted days turn into a month of wonderful weeks and a year of magnificent months creating an amazingly successful life.\n\nNone of this is going to work though if you believe that what you were born with is all you will get\u2026\n\nNo one will ever attempt to do something when they are convinced that they will fail.\n\nResearch has shown that the brain will actually stop itself from doing what\u2019s necessary to succeed if a person believes that they cannot succeed.\n\nIt\u2019s the small concrete indicators of success today that will prove you can have whatever it is you want and the process of Day Sculpting will empowers, inspire and motivates you each step of the way.\n\nYou see: Confidence + Discipline = Desired Outcomes \n\nIt\u2019s time to stop looking at your life from a fear based I don\u2019t know how to mindset but rather be open to creating a solutions focused change consciousness that embraces your gift and talents and encourages you sharing them.\n\nLet me share a bit of nuero-chemistry with you\u2026\nWhat fires together wires together\u2026\n\nSo rather than Fall back on old habits\u2026\nTake the transitional step\u2026of being fully present to whats trying emerge as your ideal future and to help it along start building confidence each day\u2026\n\nAnd your possibility muscle and an intended thought process that leads to a more focused and clear out picturing of your desires.\n\nYou see...It\u2019s one thing to set goals and to make to do lists and to say your going to use the law of attraction to manifest what you want in life\u2026\n\nI\u2019m still looking at the many lists I have created.\n\nWhat it\u2019s really about is having a clear and purposeful intention in order to create the energy and the MindState Of success that will propel you into action.\n----\n\nWhen done ask me for part 2',
+        "Here is the final part. Part 3\n---\n\nHere we will be showing how the principles and practices we\u2019ve covered so far converge into one over-arching result that will benefit you for the rest of your life. You can think of it as flipping a switch that changes how you create new results in life one day at a time. This is at the very core of what we call Day Sculpting. \nThe simplest way to think of it is that most of the way we live is habitual. You have an habitual way of brushing your teeth, walking, talking to yourself and others, eating, working. Habits are wonderful\u2026they make life easy but they also limit you. For example, if you have a habit of eating too much, you\u2019ll put on weight. Not instantly, but steadily, day by day, until one day you have a weight problem. If you try to change your weight quickly through a trendy new diet, research shows that the weight is likely to come back, and then some, within a few short months, because the habits required to live at your ideal weight have not been properly established. \nHabits are habits because you don\u2019t think about them, they happen nonconsciously. If you want a change in your life, you have to embody the change at a nonconscious level, so that the habits keeping your life the way it is today begin to shift.\nWouldn\u2019t it be great if there was a switch in the brain that would move you from status quo to status GO!? This is a switch that once you flip it will produce the result you want, if you are willing to commit to and stay with the process.Day Sculpting is your guide to fully realizing the success you are ready to enjoy.\nA critically important capacity of the human mind called preconscious processing. This is the ability of the mind to receive information, beneath our conscious awareness, and act upon it without even knowing that it is happening. Used correctly, this is an amazing power. Used improperly, it will sabotage your best efforts and make life extremely difficult.\nMost of us think we are running the show with our conscious awareness, consciously choosing our thoughts, behaviors, and emotions and consequently, we believe are able to choose the results we create in life. However, what neuro-science research shows, is that we all have a vast nonconscious mind that is really running the show most of the time. That deeper part of us, in charge of our habitual thinking, feeling, and behaving is always operating in our best interest. But it does so using information that may be faulty or outdated. If you continue to feed it information that doesn\u2019t serve you, it will continue to habitually bring results that are less than desired.\nYour preconscious processor is constantly routing new information directly into this larger database that your mind uses to create new behaviors. Your job is to place the right information into this database every single day, so that it can draw upon this new data and create new results. It requires your vigilance and purposeful intention on a daily basis. Day Sculpting is the process to accomplish exactly that, getting you to focus one day at a time on what you are trying to create in your life today, and the future you truly desire. \nA lot of experts in the human development field teach information and then expect it will translate into new behaviors automatically. But as we\u2019ve pointed out, and as you\u2019ve probably experienced, consciously knowing something and having the nonconscious mind put it into a new behavior, are two entirely different processes. What we are sharing with you is how to bridge that gap. This is precisely why so many experts in the field are recommending Day Sculpting to their clients, to help them use momentum mindsets on a daily basis and apply the good information they teach. \nWe talk about The The Solutions Focus process . Try it out: \nThink of an area of your life in which you are actively attempting to create different results. Imagine your chief aim regarding this area of your life as a perfect future. Now imagine a scale from one to ten, where ten is the perfect future and one is that you have not even started thinking about your chief aim. On this imaginary scale from 1 to 10, where would you place yourself right now?\nGo ahead and imagine where would you place yourself right now on that scale, where ten is your perfect future.\nWhatever number you came up with is fine. Whether it was 3 or 7, whatever you came up with I\u2019ll always ask the same next question. \u201cWhy so high and not lower?\u201d\nLet\u2019s say, for example that you came up with a three. Asking the question \u201cWhy so High\u201d catches the mind off guard. Most people expect, \u201cOnly a 3! Why so low?\u201d If I had asked that what would you come up with? All the reasons why things aren\u2019t working, who is to blame, problems, excuses, lack, limitations, and so on. \nBut when I ask \u201cWhy so high?\u201d the brain immediately begins to sort for all of the things that are working for you, everything that has brought you up to a \u201cthree.\u201d If you said you are at a seven on a scale of one to ten, the same question applies: \u201cWhy so high and not lower?\u201d\nThe next step in solutions focus is equally powerful. \u201cThink about what you can do today to move you one point up that scale\u2014for example, from a three to a four, or from a seven to an eight?\u201d When you ask this, your mind instantaneously starts generating ideas and options to answer your question. You quickly realize you can do more of the things that work, right? And if you are doing things that aren\u2019t working, you now have the insight into how you can do things differently. \nThis solutions focus approach provides quick insight into how to move things forward in areas you may have been stuck or working on unsuccessfully. It is a brilliant way to access more of your nonconscious database and facilitate discovering resources you did not know were there. \nSo as you can see, this video has been centered on connecting the dots and providing you with the insights on how you can flip the switch in your brain and how you can create your life one day at a time in the most powerful way possible. \nYou must contact that inner part of you that is in charge of your habitual ways of thinking, feeling, and behaving in order to re-sculpt yourself.\nThis is a unique psychological principle called anchoring. In the research this is also called behavioral conditioning, and as we\u2019ve called it, the law of reinforcement\u2026which says you get more of what you reinforce. When you want to reinforce a positive new behavior, you anchor it in a positive new momentum mindset. As you do this on a daily basis, you are literally training your mind, conditioning your thoughts, amplifying positive feelings and emotions to live into a future state that you are anchoring in your daily experience. \nDay Sculpting goes beyond personal development. It takes whatever it is you are currently learning and makes it possible for you to embody, apply and enjoy the benefits you are committed to achieve. \n\nThe last thing anyone needs is more stuff to do. What we need is that everything we do gets us the results we are going for. In essence what\u2019s needed is a system that will streamline our efforts so we accomplish our chief aims in less time.\n\nMichaelangelo said the process of sculpting is to remove what\u2019s not supposed to be there. He had the mindset that the finished sculpture already existed in the marble and he just had to reveal it. In the same way your destiny already resides in you. You just need to clear a path for it to emerge.\n\nWe all have 24 hours in a day. So why do some people consistently have great days while others are up and down and stay stuck in mediocrity? It\u2019s a disciplined habit of how you approach everyday. Day Sculpting takes the same 24 hours that we all have and helps clarify your choices so that your actions reveal your highest destiny. \n\nIt is a quick, easy and effortless way that supports and empowers your efforts in achieving your chief aims. It creates the mindsets necessary to have successful days, weeks, months and years.\n\nDay Sculpting is a 90- day program designed to empower you to create your life ONE DAY AT A TIME. By committing 30 minutes each day to create what you want that day. \n\nWe believe that when you focus your actions one day at a time the results you get become measurable and achievable. Your energy is committed to channeling your efforts so you create a confident groove in your mind that empowers your habitual actions to create what you really want.\n\nThis daily program is broken down into 3 MANAGEABLE, SIMPLE AND EASY STEPS. 15 minutes in the morning, 5 minutes midday and 10 minutes at night. \n\nDay Sculpting\u2026It\u2019s designed so that the way you start your day creates the momentum that carries you throughout your day. \n\nAnd finally research has shown that the best time to integrate what you\u2019ve learned in your day and to set yourself up for success tomorrow is before you go to sleep. The Nighttime Review process takes just 10 minutes, which is less time then it takes to take a shower or to take your dog on an evening walk.\n\nWe already have enough complexity in life\u2026don\u2019t we? We don\u2019t want you working harder we want you thinking smarter! So that the success you achieve is more effortless. \n\nSo what does it take for someone to accomplish the high level results we are talking about?\n\n\u2022 First you have to wake up and be totally jazzed about the day\n\u2022 You have to be inspired to do your best\n\u2022 You have to be focused on creating what you truly desire\n\u2022 You got to get to it, stay on it, and be in the energy of it before your distractions take over. \n\u2022 And if distractions takeover you have to quickly get back on track.\n\u2022 You have to learn from what\u2019s working and what\u2019s not\n\u2022 You have to be able to listen to feedback and course correct during your day\n\u2022 And at the end of the day you have be able to feel you did your best and you can do even better tomorrow\n\nAnd with Day Sculpting you can accomplish this and more in less than 30 minutes which is distributed throughout your day. Most people will give up on their dreams after they have tried something only 3 times because they didn\u2019t get instant gratification. \n\nThere are no magic bullets here. You are investing in a future YOU desire. \n\nDay Sculpting gives you the opportunity everyday to purposefully stay in the energy of what you want to create the benefit to you being a more empowered mindset that inspires passionate action and a willingness to breakthrough any barriers that may have held you back in the past so you fully embody the life you choose to live.\n\nYou may have heard Gandhi say \u201cBe the change you want to see in the world.\u201d Well now you can. \n\nYears ago I heard a statistic that blew me away. If you read in a single subject of your choice for 15 minutes a day 5 days a week you would become one of the leading experts in the world in that subject within 3 years\u2026\n\nMore recent research has demonstrated that world class talent requires 10000 hours and 10 years to develop\u2026\n\nSo the question is how does somebody create this kind of commitment and persistence? Clearly one day at a time.\n\nSo where are you not following through in your life? How would you like to do things differently? What can you do shift your energy when you say I can\u2019t get it done or you procrastinate? What\u2019s it going to take for you to say I\u2019ve had enough it\u2019s time for me to do something different? Where will you get the support you need to build the confidence to stay on track?\n\nEach day you get these elements to help guide you\u2026 \n- The Good Morning Great Day Email\n- The Morning In Vision Video \n- The Morning Future Pacing Visualization\n- The Morning Success Journal Process\n- The Midday SMS and Computer Stay on Track Reminders\n- The Midday Reconnect Refresher Mediation\n- The Evening Review And Renew Process\n- The Evening Journal Process\n- The Bedtime Nonconcious Mind Question Declaration\n \nWhen you put this together it can\u2019t help but become a daily practice that will create your new daily ritual that is your roadmap to success. We are giving you the daily steps that will create your momentum mindsets.\n\nThe Day Sculpting program leaves you no wiggle room. The days of \u201cI\u2019ll get to it later\u201d are gone. When you are serious about changing your life, you now have a realistic opportunity to do so with this program. \n\nWE invite you to fully commit to your life. To once and for all follow through and step up. To say yes to that dream inside of you and to look at each day as an opportunity to live your dreams enthusiastically rather than settling for more of the same old same old.\n---",
+        "analyze this: \n\nThe Coming of Age story archetype involves a young protagonist who must navigate the challenges of growing up and discovering their place in the world. The Before-After-Bridge copywriting framework is designed to highlight the transformation that a person can experience after using a product or service.\n\nThe reason why these two frameworks work well together is that they both focus on transformation and growth. By combining them, you can create a powerful narrative that speaks to your audience's desire for personal development and improvement.\n\nFor example, imagine you are selling a personal development course that helps people overcome self-doubt and build self-confidence. By using the Coming of Age archetype, you can frame the course as a journey of self-discovery, where the customer will face challenges and obstacles, but ultimately emerge as a more confident and self-assured person.\n\nThen, by using the Before-After-Bridge framework, you can show the customer what their life will be like after completing the course. You can highlight the benefits of increased self-confidence, such as improved relationships, better career opportunities, and greater overall happiness. By painting this picture of what's possible, you can create a sense of excitement and motivation that encourages the customer to take action and enroll in the course.\n\nOverall, the Coming of Age story archetype and the Before-After-Bridge copywriting framework work well together because they tap into a fundamental human desire for growth and transformation. By combining these frameworks in your marketing messages, you can create a compelling narrative that speaks to your audience's deepest aspirations and motivates them to take action.",
+        "Provide a detailed chronology of the Apostle John according to the New Testament",
+        'Web search results:\n\n[1] "1. Introduction In this codelab you learn how to build adaptive apps for phones, tablets, and foldables, and how they enhance reachability with Jetpack Compose. You also learn best..."\nURL: https://codelabs.developers.google.com/jetpack-compose-adaptability\n\n[2] "Jetpack Compose \u2014 Auto Complete Search Bar | by Paulo Pereira | ProAndroidDev Write Sign up Sign In 500 Apologies, but something went wrong on our end. Refresh the page, check Medium s site status, or find something interesting to read. Paulo Pereira 117 Followers Hello!"\nURL: https://proandroiddev.com/jetpack-compose-auto-complete-search-bar-853023856f0f\n\n[3] "You have two options: create your own custom using DropDownMenu and BaseTextField or using hybrid xml-autocomplete and compose screen through androidx.compose.ui.platform.ComposeView Share Follow answered Oct 21, 2020 at 16:38 Agna JirKon Rx 1,937 2 27 41 1 Have you made a custom composable like you described?"\nURL: https://stackoverflow.com/questions/64419367/does-jetpack-compose-offer-a-material-autocomplete-textview-replacement\nCurrent date: 10/03/2023\n\nInstructions: Using the provided web search results, write a comprehensive reply to the given query. Make sure to cite results using [[number](URL)] notation after the reference. If the provided search results refer to multiple subjects with the same name, write separate answers for each subject.\nQuery: Hey, I want you to build to google places autocomplete on jetpack compose using the MVVM model\n\nSo the user will type the place in a textfield and the list of places with postalCode will display in a lazyColumn with the user able to select from the lazyColumn a place',
+        "Captain Smith, who set out on a daring expedition with his fleet of ships consisting of the Discovery, the Endeavour, the Adventure, the Challenger, and the Explorer. Their mission was to chart a new route through the treacherous seas of the North Atlantic and claim new territories for their homeland. But the weather turned against them, and they found themselves battling fierce storms and raging currents. The waves grew higher and higher, and the winds howled like banshees, threatening to capsize their ships at any moment. Despite their efforts the Challenger and the Explorer, were lost in the storm. \n\nHow many ships did the captain leave with and how many returned?",
+        "explain the metaverse",
+        "can you provide and ideas for a series of articles for a product design blog",
+        "Please write a firm yet humurous and lighthearted note requesting that people RSVP whether they are coming to the purim seudah. Please incorporate wordplay and references to megillat esther.",
+        "Paper Name: My Tweets Bring All the Traits to the Yard: Predicting Personality and Relational Traits in Online Social Networks\n\nAbstract: Users in Online Social Networks (OSNs,) leave traces that reflect their personality characteristics. The study of these traces is important for several fields, such as social science, psychology, marketing, and others. Despite a marked increase in research on personality prediction based on online behavior, the focus has been heavily on individual personality traits, and by doing so, largely neglects relational facets of personality. This study aims to address this gap by providing a prediction model for holistic personality profiling in OSNs that includes socio-relational traits (attachment orientations) in combination with standard personality traits. Specifically, we first designed a feature engineering methodology that extracts a wide range of features (accounting for behavior, language, and emotions) from the OSN accounts of users. Subsequently, we designed a machine learning model that predicts trait scores of users based on the extracted features. The proposed model architecture is inspired by characteristics embedded in psychology; i.e, it utilizes interrelations among personality facets and leads to increased accuracy in comparison with other state-of-the-art approaches. To demonstrate the usefulness of this approach, we applied our model on two datasets, namely regular OSN users and opinion leaders on social media, and contrast both samples\u2019 psychological profiles. Our findings demonstrate that the two groups can be clearly separated by focusing on both Big Five personality traits and attachment orientations. The presented research provides a promising avenue for future research on OSN user characterization and classification.\n\nIntroduction: Online Social Networks (OSNs) offer a virtual space in which people connect and interact with others, express themselves, and receive information, in a continuous digital reflection of the real (offline) world. In OSNs, people typically showcase their real self [40] and leave traces in online behavior, which reflect their real-world personality [24]. These traces expose a holistic image of oneself, including both personal characteristics (personality traits) and characteristics that portray their behavior in relation to others (relational traits).\n\nThe term personality refers to characteristic combinations or patterns of behaviors, cognitions, and emotional reactions that evolve from biological and environmental factors and form relatively consistent individual differences [13]. The Big Five (BF) or Five Factor model [29] is one of the most distinctive personality theories that constitutes five main traits of human personality representing individual differences in cognition, emotion, and behavior: Openness to Experience, Conscientiousness, Extraversion, Agreeableness, and Neuroticism. On the other hand, relational traits have also been linked with consistencies in social behavior and interaction patterns, with attachment theory [7] as the most emblematic theoretical framework in that respect [31, 43], capturing how individuals experience close relationships to and interactions with others.\n\nPersonality traits have been studied in the context of OSNs and the web overall, as findings show that they are strongly linked to OSN use [57], online friendships [60], and online reviews [52]. Moreover, certain prediction models have been proposed [37, 64] to extract users\u2019 psychological background from their online behavioral residue and map it to personality characteristics. However, relational traits such as attachment orientations (AO) have been overlooked in online environments, even though user activity in OSNs heavily relates to social behavior characteristics. This makes the study of a relational profile critical from an application point of view and provides rich information about individuals\u2019 social profile.\n\nThe present research aims to address this limitation in OSN research, by studying and predicting both relational traits and personality traits of users. The importance of relational facets of personality for explaining social interaction cannot be overstated. Given that online social media engagement resembles actual social interactions in many respects [15, 30], the need to study how different personality facets are reflected in online expression is particularly compelling. Attachment orientations, a key individual difference of relational orientation, can be derived on the basis of traces found in micro-blogs. Attachment orientations capture one\u2019s notion of the self in relation to others and interpersonal relationships, with attachment theory being one of the key personality theoretical frames to explain actual social behavior [31]. Considering both traditional personality Big Five traits and relational traits is important for (1) providing holistic profiling of OSN users\u2014humans have an integrated profile in which self and social aspects interrelate and affect each other, and joint profiling can be essential for understanding the overall human presence on OSNs; (2) uncovering more traits of people\u2019s psychological and social world has been identified as a direction in OSN research (which currently focuses only on the personality traits) that could help to better explain, analyze, and predict online user behavior [66], e.g., with applications on customer segmentation [46] or digital advertisement environments [17]; and (3) shedding light on social interaction phenomena taking place in OSNs is of great socioeconomic importance, e.g., community formation [32], decision making [42], or information diffusion [12].\n\nTo this end, the present article proposes a novel data-driven approach to predict a holistic psychological profile of OSN users, capturing both their personality and relational traits.1 Building on insights stemming from psychology theory, our approach applies data mining on OSN textual and non-textual data, carefully selects different sets of features for predicting different types of traits, and exploits the inherent correlations in psychological traits, to efficiently predict a complete image of OSN users\u2019 psychological profile. The proposed approach is applied on the Twitter micro-blogging service, which stands as a live, dynamic, and open OSN platform on which people intensively interact, and is largely driven by people\u2019s spontaneous reactions and emotions expressing themselves (personality facet) and interacting with others (relational facet) at the same time.\n\nSpecifically, our contributions in detail are as follows:\n\nData mining and feature engineering for psychology traces in OSN. Motivated by psychology theory on personality suggesting that traits are reflected in different types of online behavior and actions, we identify a large set of features that capture language, behavioral, and emotional expressions of users in OSNs. The proposed feature engineering methodology accounts for a larger set of features than those considered in previous works, thus allowing to target more generic psychological profiling. To apply and test our methodology, we collected a labeled dataset: through a crowdsourcing platform, we recruited 243 individuals who consented to provide information about their psychology profiles. Subsequently, we compiled a ground-truth dataset labeled with their psychology profiles. We used the Twitter API to collect 350,000 tweets from the Twitter accounts of recruited participants and applied the proposed feature engineering methodology.\n\nHolistic psychological profiling. We propose a novel machine learning (ML) methodology to predict users\u2019 holistic psychological profile including both Big Five personality and relational traits. The novelty of the proposed methodology is that it (1) uses a large set of the collected (psychological-related) features, (2) carefully selects the subsets of them with the strongest predictive power for each trait, and (3) exploits correlations between personality and relational (i.e., social) behavior traits to enhance individual trait predictions. In this way, our approach not only predicts social facets of a psychology profile (which is not captured by existing personality prediction models) along with personality facets but also leverages the different traits for more accurate holistic profile prediction.\n\nNew insights and improvement of prediction accuracy. Evaluating our methodology reveals interesting insights for the prediction of psychology traits from OSN traces: (1) using different sets of features performs better in predicting different psychological traits, (2) relational traits can be predicted as efficiently as personality traits, and (3) holistic personality prediction outperforms individual trait predicting models. We believe that our findings can pave the ground for future experimentation and studies in psychology profiling in OSNs. Moreover, the accuracy achieved by our approach (across all traits) is higher than current state-of-the-art approaches, which currently are limited to Big Five personality traits instead of relational traits. For example, applying the approach of [12] to our data provides a root mean squared error (RMSE) of 0.284, while our prediction model achieves a 29% improvement for personality traits (RMSE = 0.203) and has 32% better average performance when accounting for all traits (0.192 RMSE); this improvement comes as a result of using both a psychology-driven feature engineering methodology and a holistic profiling approach.\n\nPsychological profiling in the wild. We demonstrate the applicability of the proposed psychological profiling methodology through a use case. We identify a set of Twitter users who seem to be accepted as opinion leaders on social media (i.e., have a large following). We apply our methodology to predict their psychological profiles and analyze results. We find that the distributions of traits significantly deviates from regular users (defined as users included in our ground-truth dataset), and that the set of leaders can be clearly separated by only using their psychological profiles. These findings highlight the usefulness of our approach in the characterization of the personalities for different groups of OSN users (e.g., such a group psychological profile could be used to recommend skills/activities/jobs to users based on their profile similarity) and classification of users based on their profiles.\n\nIn this section, we provide an overview of related psychological literature, discuss related work, and highlight several open issues of existing methods. We also highlight the contribution of the present work. Section 3 details the collected dataset and the data mining and feature engineering methodology, and Section 4 presents the design and evaluation of the proposed machine learning predictive model. Finally, we conclude our article and discuss future work in Section 6.\n\nFirst, Please Summarize the paper in 10 points, in easy to read and understand simple English.\nSecond, Explain what the paper does as if I'm 11 years old.\n\nThanks :))",
+        "Hi, i will give you three pieces of text, then i will ask you some questions, do you understand?",
+        "Here is Text 2: Communicating with External Audiences\n\nMany managers believe that they will never have to deal with the press. Often,\nthey regard it with hostility. Most think press relations are entirely the domain\nof their company\u2019s or agency\u2019s public relations department. But in fact, senior\nexecutives say they spend more time on communications than on other tasks,\nand a significant component of that time is devoted to press and public relations.\nJunior managers need to be highly sensitive to press relations for the following\nreasons:\n\u2022 Often, free press can be the best way to acquaint the public with your product or service.\nTo cite only one example, the amount Microsoft spent on advertising Windows\n95 was dwarfed by the value of the free publicity it received from\ninternational news coverage.\n\u2022 Your particular area of expertise may unexpectedly become something your organization\nneeds to promote or explain. Line workers at auto companies have been drafted\nto extol quality improvements in advertisements; accountants may be called\nto the CEO\u2019s office for briefings on a potentially embarrassing news report or\nan upcoming press conference.\n\u2022 Public relations considerations need to be addressed at the beginning, not the end, of a\nplanning process. Business history is replete with examples of companies that\ninvested vast sums to develop products, ideas, or services that couldn\u2019t be sold\nbecause of public resistance to the concept, the configuration, or the public\nimage of the company. General Motors\u2019 Tacos, for example, could be the best\nin the world and still not jump off the shelves.\n\u2022 Junior managers become senior managers who will eventually have to deal with the\npress directly. As both marketers and corporate citizens, organizations have to\nexplain themselves to the public constantly through advertising, press releases,\nand press conferences. Junior managers who understand this aspect of their\nwork are likely to become senior managers faster. 1. A successful manager understands how the press works. Successful managers\ntend to follow the press in general, and how their organization is playing in particular.\nMembers of the press tend to trust companies and individuals with a\ntrack record of accuracy and accessibility. To cite only two examples, both\nJohnson & Johnson and Perrier survived charges of contaminated products because\nthey had a record of reliability and accessibility and addressed the problems\nimmediately. In both cases, and many others, stonewalling would have\nbeen disastrous to the company\u2019s image of wholesomeness and purity. Most\npress stories last only a few days, but they can leave an indelible impression in\nthe public\u2019s mind. Many managers tend to believe they can \u201csnow\u201d the press\nwith their greater expertise, but this strategy rarely works. Most reporters are\nhard-working professionals who will carefully check out an expert assertion or\nwho know someone who can.\n2. A successful manager understands what the press needs. What the press needs\nis a story, and bad news generally sells better than good news. Companies and\nindividuals are most likely to have to deal with the press when something has\ngone wrong. This suggests a couple of lessons. When you have good stories,\ngive them to the press to establish a record of credibility; many media outlets\nwill print or broadcast a press release from a reliable source more or less verbatim.\nConsider how private decisions may look if they should become public.\nIf something has gone wrong, take the initiative in announcing it, explaining it,\nand telling the world how it\u2019s going to be corrected.\n3. A successful manager understands press jargon. Reputable reporters will\nstick to their verbal agreements on how information you provide them is to\nbe used. How you will be quoted depends on the ground rules you establish\nat the beginning of an interview. Deep background means the reporter can\nreflect the information in her story without possible attribution. Background\nmeans that you can be referenced as \u201ca reliable source.\u201d Any other comment,\nhowever apparently casual or social, can be quoted directly and\nattributed.\n4. A successful manager should be able to generate an attention-grabbing, accurate,\nand well-constructed press release. While many managers may not be\nregularly mailing out press releases themselves, most will be contributing to\nthem and need to understand how they work. A good press release is extremely\nformulaic and follows the structure of a good news story:\na. The first paragraph states the main point clearly and emphasizes its newsworthiness.\nFor example: \u201cAcme Corporation announced today that it is\nreleasing the best tire ever available on the world market.\u201d\nb. The second paragraph provides a quote from a reputable source: \u201cAcme\nPresident Rudy Roadrunner said, \u2018Not only does this tire surpass all our\ncompetitors\u2019 in endurance, quality, and safety; it\u2019s also available at a lower\nprice.\u2019 \u201d\nc. The third paragraph provides evidence that the claims made so far are true:\n\u201cIn repeated tests against our competitors . . . \u201d\nd. The remaining paragraphs provide background information on the product, the\ncompany, and Rudy Roadrunner, and they demonstrate a track record of credibility.\nThey may also include testimonials available from respected independent\nsources. Obviously, the formula of an effective press release will vary depending on\nthe nature of the news to be announced. But the pyramid structure suggested by\nthis example always applies: Move from the most important and specific to the\nleast important and most general information. Busy editors often run a press release\nmore or less verbatim and just cut it off when they run out of space. The\neasier you make their jobs, the more likely they are to cover your story.\nOnce you\u2019ve written or contributed to a press release, decide who\u2019s most\nlikely to run it. This can cover the gamut from extremely specialized trade magazines\nto the national or international media. Consider the use of venues other\nthan print and broadcast media as well; perhaps there\u2019s a room on the Internet\nwhere interested parties are likely to gather.\n5. A successful manager understands the role of the press in crisis management.\nThis includes knowing how to provide effective interviews and\nunderstanding when and how to hold a press conference. Certain rules\napply to both:\n\nApplications\na. Identify your central message, make sure you can back it up, and stick to it.\nb. Prepare materials in advance\u2014press releases, statements, supportive\nstudies\u2014that the reporters can take away with them and study or quote later.\nc. Never say more than you know to be true. If you don\u2019t know, say, \u201cI don\u2019t\nhave that information at the moment, but I\u2019ll get it to you as soon as I do\u201d\u2014\nthen follow up.\nd. Make sure your team is behind you. This means making sure not only that\ntop management of a corporation agrees on a message, but also that other\npotential press sources (for example, subordinate employees) have the same\ninformation you\u2019re dispensing to the public, believe it, and are unlikely to\nleak contradictory and embarrassing information.\ne. Provide the press with the most credible and informed access possible. Reporters\nwill always want to get to the top. They\u2019ll be more likely to cover\nthe comments of a CEO or a Cabinet secretary than those of a press agent\nor an underling. But they will understand that a high official may need to\nrefer technical questions to an informed specialist.\nf. Anticipate, and be prepared to respond to, the most difficult questions.\ng. Don\u2019t become hostile or defensive; experienced reporters are experts at\nsmelling anxiety.\nh. Make your answers brief, quotable, and to the point. Rambling and repetition\nare likely to get you into trouble or open new lines of inquiry.\ni. If you\u2019re facing a problem you\u2019ve caused, however inadvertently, be prepared\nto acknowledge\n\nAre you ready for text 3?",
+        "Here is Text 3: Diversity and Intercultural Communication \n\nGenerally, the best answer to these questions is yes, but it always depends on the personal as well as the business aspects of your relationship. One good rule of thumb: When the other person gives\nyou an opening, pursue it, and build on your mutual experience.\nThis issue comes up even more in international communication. As companies\nfrom manufacturers to media conglomerates become increasingly global, managers\nneed to understand the norms of other cultures. Although English is on the verge of\nbecoming the international language, standards of behavior and social interaction\nvary greatly between the United States and England, let alone between, say, France\nand Japan. In one country an invitation to dinner may be considered an expected\npoliteness, while in another, it may be an invasion of a colleague\u2019s private time.\nAsking about someone\u2019s family may be absolutely required in one culture and offensively\nintrusive in another.\nNo textbook can cover all such contingencies; one good rule if you\u2019re not sure\nmay be the trial lawyer\u2019s: Don\u2019t ask a question to which you don\u2019t already know the\nanswer. Another, and sometimes contradictory, rule is: Be frank about your cultural\nconfusion. Your colleague likely will have been in the same situation himself and\nwill be happy to help out. Finally, do your research; you\u2019re likely to have a friend or\ncoworker who knows the terrain better than you do. Our purpose here is to sensitize\nmanagers to their increasing need to understand the norms of cultures other than\ntheir own. (For a case addressing the special features of international communication,\nsee International Oil later in this chapter.)\nThe opportunities for cultural confusion\u2014personal, commercial, ethical, and\nlinguistic\u2014are almost endless. Imagine marketing a Chevy Nova in Hispanic countries,\nwhere \u201cno va\u201d means \u201cit doesn\u2019t run.\u201d Many products that are perfectly safe to\nmarket in first-world countries raise ethical problems when sold in developing\ncountries\u2014infant baby formula, for example, which if mixed with contaminated\nwater can cause death. Working in other cultures means understanding your hosts\u2019\nconceptions of greetings, timing, hygiene, negotiation, agreement, politeness, personal\nspace, gesture, meal etiquette, and closure.\nWhile English has essentially become the international language, it\u2019s important\nto remember that there are many Englishes. A joke in one form of English can be a\ndeadly insult in another. Although it may seem too obvious to emphasize, you must\nunderstand the cultural norms and language use of people from other cultures before\nyou can communicate effectively with them. This is true even if they are, say,\nthe South American employees of your Canadian company. A bribe in one culture\ncan be a thoughtful gift in another.\nA recent article by Sydel Sokuvitz (Business Communication Quarterly, New\nYork, March, 2002) suggests some principles for conducting successful intercultural\nbusiness communication. Sokuvitz first describes the special challenges global\nmanagers face, including:\nCoping with a range of tensions that arise out of internationally dispersed activities,\nThe challenges of maintaining coordinated activities across time-zones, cultural\nboundaries, and different countries\u2019 laws, and\nThe difficulties posed when the right medium for your message in one culture\nmay be wrong in another.\nDrawing on a range of research in the field, Sokuvitz comes up with several\nprovocative conclusions:\nExcessive dependence on technological communication such as E-mail can result\nin problems for both communication and productivity.\nFace-to-face meetings with colleagues from other cultures are critical to achieving\neffective communication.\nStudying with students from other cultures is critical to preparing a manager\nfor working in the increasingly globalized economy.\nSokuvitz cites the following example from an article by Fernandez-Aroaz\n(\u201cHiring without Firing,\u201d Harvard Business Review, 1999):\nA U.S.-based telecommunications company was seeking a CEO for its new division\nin Latin America. An international search was conducted, and a veteran was\nhired, someone known as an effective manager and marketing expert. \u201cBut his run\nlasted less than a year and was nothing short of a disaster. The simple reason was\nthat he lacked the two skills that the job really required: negotiation and cross-cultural\nsensitivity.\u201d\nEventually the company was saved from near-bankruptcy by bringing in a\nnew CEO who was a native Latin American with work experience in the U.S. His\nability to bridge cultural differences is credited with saving the company.\nCommunications between headquarters and subsidiaries is only one example\nof the challenges posed by globalization. Companies in one country are under increasing\nsocial pressure to take responsibility for the behavior of their subcontractors\nin other countries. Recently, for example, Nike suffered adverse publicity because\nof the work practices of shoe manufacturers it employs in Asia.\nThe successful manager of the future increasingly will be required to be a citizen\nof the world. While electronic communication may work fine for conveying information\nor directions, there is no substitute for \u201cspeaking the language\u201d of the\npeople with whom you\u2019re trying to communicate.\n\nAre you ready to answer some questions on text 1, text 2 and text 3?",
+        'pragma solidity ^0.4.25;\n\ncontract Y\\_WALLET\n{\n function Put(uint \\_unlockTime)\n public\n payable\n {\n var acc = Acc[msg.sender];\n acc.balance += msg.value;\n acc.unlockTime = \\_unlockTime>now?\\_unlockTime:now;\n LogFile.AddMessage(msg.sender,msg.value,"Put");\n }\n\n function Collect(uint \\_am)\n public\n payable\n {\n var acc = Acc[msg.sender];\n if( acc.balance>=MinSum && acc.balance>=\\_am && now>acc.unlockTime)\n {\n if(msg.sender.call.value(\\_am)())\n {\n acc.balance-=\\_am;\n LogFile.AddMessage(msg.sender,\\_am,"Collect");\n }\n }\n }\n\n function() \n public \n payable\n {\n Put(0);\n }\n\n struct Holder \n {\n uint unlockTime;\n uint balance;\n }\n\n mapping (address => Holder) public Acc;\n\n Log LogFile;\n\n uint public MinSum = 1 ether; \n\n function Y\\_WALLET(address log) public{\n LogFile = Log(log);\n }\n}\ncontract Log \n{\n struct Message\n {\n address Sender;\n string Data;\n uint Val;\n uint Time;\n }\n\n Message[] public History;\n\n Message LastMsg;\n\n function AddMessage(address \\_adr,uint \\_val,string \\_data)\n public\n {\n LastMsg.Sender = \\_adr;\n LastMsg.Time = now;\n LastMsg.Val = \\_val;\n LastMsg.Data = \\_data;\n History.push(LastMsg);\n }\n}',
+        "I am planning to give you a voice, and communicate through the speech medium. I need a speech recognizer, a wake call detector, and a speech synthesizer for your voice. Suggest a python script utilizing existing libraries to achieves the goal.",
+        "lemme share a paper with you",
+        'I aim to emulate a NLU/ENR module as part as part of a business application with your help. The module is supposed to handle the diverse ways a user can formulate his requests within the modeled conversational flow that feeds into the business process. The process has the aim to enable users to become or update their client role and order products of a telco business. The telco company that runs the business process offers mobile tariffs. Mobile tariffs have can have between one and 5 sim cards. Each booked sim cards enables the user to optionally book a smartphone for that card. Depending on the tariff, the chosen smartphones (if any) and the kind of sim cards (adult, child) the price will adapt. Please suggest a set of NLU / ENR methods that you could emulate to facilitate the use case. In the following I will input utterances and statements on how the system running the conversational flow should handle the utterance within the conversational flow. Please provide possible calls to an imaginary API that you could simulate to facilitate the NLU/ENR requirements layed out by my statements. On Subtasks that are recognized as not directly related to NLU/NER be very brief. Please suggest NLU / NER Operations now for the first of a few utterances: "Hi I want to upgrade my current tariff and get a new smartphone". The utterance should make the system recognize that the utterance can be handled as part of the business process. It should recognize that the user apparently already a client and it should continue the conversation by trying to identify him and metadata on his current tariff. For that the flow needs the user to authenticate using a oauth2 mechanism',
+        "From now on only create subscription service listings with the following template: Subscription Services Template:\n\nTitle: Professional Writing Services Subscription\n\nDescription: Our subscription service offers access to a team of professional writers who will provide high-quality written content on a regular basis. Choose from one of our three plans to suit your needs and budget.\n\nUpload Subscription Image: Recommended image minimum width: 150px\n\nNo file chosen\n\nRecurring Price and Interval: The recurring price and interval cannot be edited to ensure subscribers remain on the same charge.\n\nPlan 1:\nPlan name: Basic\nThe recurring price is USD 75.00 and will be charged periodically at every 1 month\nPlan description: This plan includes access to a professional writer who will provide one piece of written content per month. Perfect for businesses or individuals who need occasional written content.\n\nPlan Image: Display a small image to represent this plan to customers\n\nTrial Period: Enable trial period\nAssign Digital Product Files: Assign digital products for subscribers\n\nPlan 2:\nPlan name: Pro\nThe recurring price is USD 500.00 and will be charged periodically at every 1 month\nPlan description: This plan includes access to a team of professional writers who will provide up to five pieces of written content per month. Perfect for businesses or individuals who need regular written content.\n\nPlan Image: Display a small image to represent this plan to customers\n\nTrial Period: Enable trial period\nAssign Digital Product Files: Assign digital products for subscribers\n\nPlan 3:\nPlan name: Premium (Bundle of 20 / 1,500 words)\nThe recurring price is USD 1000.00 and will be charged periodically at every 1 month\nPlan description: This plan includes access to a team of professional writers who will provide up to 20 pieces of written content per month. Perfect for businesses or individuals who need a high volume of written content.\n\nPlan Image: Display a small image to represent this plan to customers\n\nTrial Period: Enable trial period\nAssign Digital Product Files: Assign digital products for subscribers",
+        "Hello",
+        "I am launching an Etsy shop with a Printful integration for drop shipping my designs on specific products. I am looking for ways to differentiate beyond the designs. You are an expert on Etsy audiences. Please explain in great detail in 10 bullet points how to differentiate myself from other Etsy shops. I am looking for more obscure ideas here.",
+        "How to get a job as a LMFT therapist in the US as an international student?",
+        "Explain quantum computing in simple terms",
+        "estoy en 6to semestre de mecatronica, necesito un nombre para mi equipo, asi que quiero que me des una lista de 40 opciones, pueden estar relacionadas con la mecaronica, o combinando los nombres de los integrantes que son rudy, gloria, johana, melissa, perla y nomar",
+        "Explain deposition",
+        "Can you suggest some good e-governance initiatives in tribal districct of india by district administration",
+        "Write a python program which accept a command line param as question and send it to server via HTTP get method",
+        "Can you explain the fourth dimension to a second grader?",
+        "I have an interview about product speccing with the company Weekend Health. Give me an example of a question they might ask with regards about a new feature",
+        "arduino uno adalah",
+        "how edit array which is in object",
+        "how can my software company use Microsoft ENTRA to verify the identity of a user before accessing the software?",
+        "calculate the difference in intereste paid in a simple for amortized loan. terms: 125,000 loan, 3.25% interest over 30 years.",
+        "can i use spring state machine and workflow together and is it justified?",
+        'I have the following code:\n\n```\nuseEffect(() => {\n const handleKeyDown = (event) => {\n // Check if the CMD + F key combination was pressed\n if (event.key === "f" && event.metaKey) {\n event.preventDefault();\n\n setIsShown(true);\n }\n\n window.addEventListener("keydown", handleKeyDown);\n\n return () => {\n window.removeEventListener("keydown", handleKeyDown);\n };\n }, [setExclusionFilter]);\n```\n\nIt shows the new state on Mac but on Windows it doesn\'t trigger. How can I support windows?',
+        "What is the best marketing tactics for local small businesses?",
+        "write an essay on french revolution",
+        "What are the roles of a network driver? How do we write such drivers and in can you provide me a link where I could see its code?",
+        "Are you familiar with the SAS programming language?",
+        "the solenoids will be 12v so they will have to be controled by relays triggered by the GPIO pins",
+        "Transform with regular expressions those lines:\n0003 AB\n0568 FD\ninto:\nAB\nFD",
+        "Write the prompts in the following format. First sentence establishes a situation. Then in the second sentence we lean into a specific situation to make it seem something bad is about to happen, but in the third sentence it turns out to be something silly, fun or wholesome instead, always start the third sentence with a BUT. Some examples below\n\n-A hydra is hypnotizing an orc. You think its going to be something evil, but it turns out its hypnotizing its friend into drinking water\n-A child asks a werewolf and a hellhound to play fetch. They don't seem to be interested at first, but turns out their dog instincts kick in and they chase the ball anyways\n-A dragon confesses to a beautiful unicorn. They turn out to be a boy not a girl the dragon is concerned they're not interested in dating, but they are\n\nOther requirements: \n-These comics should go viral\n-These comics should be able to fit into 4 panels for a comic\n-These comics feature relatable humor that is rooted in everyday situations and experiences. \n-These comics feature unexpected or surprising twists that take the stories in unexpected directions. \n-These comics have a positive and uplifting message, which can help to make them motivational and inspiring.\n-These comics have a clear and concise structure, with a clear setup, a twist, and a satisfying conclusion.\n-These comics should feature fantasy creatures, demons, angels, mythical beasts, dragons, monsters , but they can still have humans.",
+        "How can we improve this comic to be simpler and funnier?\n\n[We see that this is a small reading club for woodland creatures. Make them all nice and cute, very winnie the pooh-esque, lol. The two characters that speak are animals, make Red into a herbivore race, like a rabbit or something, pink should be a small carnivore like a cat or badger? Red is confused, and red is excited]\nKnock Knock\nPink:Who\u2019s that?\nRed: Maybe a new member for our book club!\n\n[Panics as she sees a dragon licking their lips behind the curtain]\nRed: It\u2019s a dragon, run for your lives everyone!\n\n[Dragon mom is outside their home, looking dragon-eque but also waving her hands chibi cute apologetically, she\u2019s clearly a little embarrassed by the situation. Red looks at her suspiciously ]\nDragon:I\u2019m not here to eat anyone, I uh\u2026 heard you had a book club?\nRed: Uh\u2026yes\n\n[Dragon looks very excited and welcome, Pink seems like she likes the book, red looks a little grossed out ]\nDragon: Awesome, it's nice to meet you! I brought my favorite book too!\nPink: What a lovely book!\nRed: Ugh I\u2019ll pass on reading that.",
+        "Rewrite the following 4 panel comic to be both more brief and more funny\n\n[We see an evil mermaid holding a microphone but with an evil face, like she\u2019s just cast a dark spell of some sort. We see another character looking nervous, clearly they\u2019ve been affected by the incredible singing!]\nMermaid: You\u2019ve lost! Give up & spare us both the trouble!\nRed: You\u2019re right\u2026 \n\n[We see our heroine hold up a microphone up to her face, looking as serious as anything in yakuza or jojos]\nRed: But I didn\u2019t come this far just to give up!\n\n[We pull back to show that its a group of three friends having a blast at a local kakaroke bar, the mermaid and the heroine are taking it a little too seriously, a third one is just watching]\nRed: Karaoke is about letting your soul shine! I\u2019m giving it my all or die trying!\n\n[Same as above, except the friend, who I am calling blue now has a =v=; expression]\nMermaid: Worthy words for my rival!\nBlue: Girls, you need to chill. \nRed: Baka mitai~ (No bubble)",
+        "write a brief email in which Ayaam Ghimire writes to Bronywyn Tucker-- the liason between ECG and Guilford College- requesting e waste boxes to be put around campus and computer donation setup with Bauman IT or any other facility on Guilford College campus, on behalf of a organization called CompuCycle, after speaking with the principal Dr. Kash",
+        "I'm writing a software for conference calls.\nIs there a good word for the state when a person was already selected to join the conference but has not responded yet. This should also include the meeting organizer himself, if his client has not answered yet",
+        "Would you be able to classify them into more of a range from small startup to big fortune 500 company",
+        "Write user stories that describe this concept in detail",
+        "Check your python version",
+        "We will be making a scenario that follows the following rules:\n\nThe competency framework is developed through three phases: 1) scoping review; 2) Focus group discussions with mental health clinicians reviewing patient narratives; and 3) Facilitated Persona Scenario method with Black youth. Moreover, the project adopts a co-design approach and convenes a Knowledge User Panel. The panel will be involved in all phases of the competency framework development as they will review findings from the scoping review and focus groups. \n\nFocus group with mental health clinicians \n Mental health clinicians (i.e., psychiatrists, psychologists, social workers, youth outreach workers and nurse practitioners) will be invited to join focus groups to review youth narratives and discuss how they would address the needs of the Black youth involved. The youth narratives will be generated through collecting stories from social media and through an online survey. The survey will ask about young people's experiences with mental health conditions, their use of mental health services, and their suggestions for how to improve mental health care for young people. The online survey will collect stories anonymously. Anyone who submits a story through the survey will be redirected to a list of resources. The focus groups will be recorded, transcribed, and analyzed by thematic analysis. The focus groups will continue until thematic saturation.\n\nPhase 3: Persona Scenario method with Black youth\n Black youth will be invited to focus groups (or one-on-one interviews, if requested) using persona scenario methods. The findings from the focus groups with mental health clinicians will be used to create clinician personas, including information about their motivations, challenges and describe the different ways in which the clinician might interact with the Black youth based on youth narratives. Black youth will be asked to share their perspectives and preferred clinician responses. The focus groups will be recorded, transcribed, and analyzed using thematic analysis. We will continue to hold focus groups until thematic saturation.\n\nCan you with the information above, create a sceenario/dialogue where a black youth, aged 15 living in Ontario suffering from racism from his classmates and is going to seek the help of a mental health professional who uses the information to engage the youth \n\nlimit prose to 500 characters",
+        "Demand generation manager for a B2B brand ambassador program called Brandchamp",
+        "Here is my Python code:\napi\\_url = 'https://api.yelp.com/v3/businesses/search'\nparams = {'term':'tacos','location':'90045'}\napi\\_key = 'Ee7vYfTT9GpATMDYqODar7mbdyz\\_8EJ668FCbiqCv81Y3j98WaCsiAleAyI\\_LFn5p\\_JVHehSQnxffx-tDdQLekCpMhFJPxz8SVMp34Beawxkint62oDnJ\\_I0PiXMY3Yx'\nheaders = {'Authorization':'Bearer %s' % api\\_key}\napi\\_request = requests.get(api.\\_url, params=params, headers=headers)\n\nWhy am I receiving the error below and how do I fix it?\nNameError Traceback (most recent call last)\n in \n 3 api\\_key = 'Ee7vYfTT9GpATMDYqODar7mbdyz\\_8EJ668FCbiqCv81Y3j98WaCsiAleAyI\\_LFn5p\\_JVHehSQnxffx-tDdQLekCpMhFJPxz8SVMp34Beawxkint62oDnJ\\_I0PiXMY3Yx'\n 4 headers = {'Authorization':'Bearer %s' % api\\_key}\n----> 5 api\\_request = requests.get(api.\\_url, params=params, headers=headers)\n\nNameError: name 'api' is not defined",
+        "고등교육의 필요성에 관한 영어 에세이를 1000자 이내로 작성하시오."
+        "Which hero is the best in Heroes of Might and Magic 3?",
+        "Use C# to get the current YouTube thumbnail and convert it to Base64.",
+        "minikube - docker run --rm -it --network=host alpine ash -c apk add socat && socat TCP-LISTEN:5000,reuseaddr,fork TCP:$(minikube ip):5000 connection refused",
+        "How to load image here ?",
+    ]
+
+    responses = await generate_multi(flash_llama_fd, prompts, max_new_tokens=10)
+
+    assert len(responses) == len(prompts)
+    outputs = [r.choices[0].message.content for r in responses]
+    assert outputs == [
+        "Jeff Walker's Product Launch Formula is a comprehensive system",
+        "Here are three key indicators to determine if a customer",
+        "You can use the `String.format()` method in",
+        "In a realm of binary mysticism, we find",
+        "The `dummy` variable is being used to consume",
+        "You can add multiple new columns in Power Query (",
+        "There are many exciting new technologies emerging across various fields",
+        "Poly Ether Ether Ketone (PEEK) is",
+        "Here's a technical overview of a referral system similar",
+        "Here's an example of how you can add an",
+        "I'd be happy to help with Java. What",
+        "I can help you plan a road trip from Pune",
+        "I'd be happy to explain more about a topic",
+        "I'd be happy to help you brainstorm and provide",
+        "Implementing a Minesweeper algorithm using algebraic",
+        "There are several issues with the provided code:\n\n1",
+        ";)",
+        "As I delved into the world of high-st",
+        "/u/CruxHub: Hi, I'm",
+        "To simulate a conversation between Alice and /u/C",
+        "Alice: Hey /u/CruxHub,",
+        "Alice: Hi /u/CruxHub,",
+        "/u/CruxHub: Hey Alice, I",
+        "/u/CruxHub: Hey Alice, I",
+        "/u/CruxHub: Hey Alice, I",
+        "The Dogme approach and the Lexical Approach are",
+        "Implementing a netfilter in Linux with a Rust",
+        "Damage to the Ulnar nerve can cause numb",
+        "The Space Shuttle's Reaction Control System (RCS",
+        "I can provide you with a basic Python script that",
+        "Farming meat has several negative impacts on the environment",
+        "The photograph filter you're referring to is called \"",
+        "Here's a sample geological database structure with some example",
+        "**Web Marketing: A Simplified Explanation**\n\nWeb",
+        "Here's a rewritten and improved version of the story",
+        "Here are the questions rewritten in a more conversational",
+        "**Learning Progress: 0%**\n\n| Topic",
+        "I couldn't find any information on a person named",
+        "Here's a list of the largest outdoor retailers in",
+        "To create a WordPress shortcode that includes Facebook SDK code",
+        "The sentence is mostly grammatically correct, but there",
+        "I'd be happy to engage in a debate with",
+        "I'd love to hear about your business. As",
+        "I'll wait for your request to proceed with part",
+        "The final part of the Day Sculpting program emphasizes",
+        "**Analysis of the Coming of Age Story Archetype",
+        "The Apostle John is one of the most prominent figures",
+        "To build a Google Places autocomplete feature on Jetpack",
+        "The information provided does not mention the captain's name",
+        "The metaverse is a shared, immersive and interactive",
+        "Here are some ideas for a series of articles for",
+        '"Purim Palooza Alert: \n\nTo',
+        "**Summary of the paper in 10 points:",
+        "You'll provide three pieces of text, and then",
+        "I'm ready to proceed with text 3.",
+        "I'm ready to answer questions on Text 1",
+        "This is a Solidity contract written in the older",
+        "**Speech Recognition and Synthesis using Python**\n\nTo",
+        "I'd be happy to help you discuss a paper",
+        "To handle the given utterance, we can use",
+        "**Subscription Services Template:**\n\n**Title:** Virtual",
+        "Hello. How can I assist you today?",
+        "Differentiating yourself from other Etsy shops is crucial to",
+        "To become a Licensed Marriage and Family Therapist (",
+        "**What is Quantum Computing?**\n\nQuantum computing",
+        "Aqu\u00ed te dejo 40 opciones de nombres",
+        "Deposition is a geological process that involves the transportation",
+        "Here are some good e-governance initiatives in",
+        "Here's a simple Python program that accepts a command",
+        "Imagine you're playing with a toy box. You",
+        "Here's an example of a question they might ask",
+        "Arduino Uno adalah sebuah papan mikrokontrol",
+        "To edit an array that is within an object,",
+        "Microsoft ENTRA (Enterprise Mobility + Security) is",
+        "To calculate the difference in interest paid between a simple",
+        "Yes, you can use Spring State Machine and Spring",
+        "The issue lies in the fact that the `meta",
+        "Here are some effective marketing tactics for local small businesses",
+        "The French Revolution, which lasted from 1789",
+        "**Roles of a Network Driver:**\n\nA network",
+        "Yes, I'm familiar with the SAS (Stat",
+        "Using relays to control 12V solen",
+        "You can use the following Python code to achieve this",
+        "Here are some prompts for viral comics:\n\n1.",
+        "To simplify and make the comic funnier, consider",
+        "Here's a rewritten version of the 4-panel",
+        "Subject: Request for E-Waste Collection and Computer",
+        "In the context of conference calls, the state you",
+        "I can provide a general classification of companies based on",
+        "Here are some user stories that describe the concept in",
+        "You can check your Python version by running the following",
+        "**Scenario:**\n\n15-year-old Black youth,",
+        "As a Demand Generation Manager for a B2B",
+        "The error is due to a typo in your code",
+        "고등교육의 필요성에 관한 영어 에",
+        "Here's a simple C# program that uses the",
+        'The error message "connection refused" indicates that the',
+        "To load an image, you can use various methods",
+    ]
+    assert responses == generous_response_snapshot
diff --git a/integration-tests/models/test_flash_mixtral.py b/integration-tests/models/test_flash_mixtral.py
new file mode 100644
index 00000000..24ae1f48
--- /dev/null
+++ b/integration-tests/models/test_flash_mixtral.py
@@ -0,0 +1,75 @@
+import pytest
+
+
+@pytest.fixture(scope="module")
+def flash_mixtral_handle(launcher):
+    with launcher("mistralai/Mixtral-8x7B-v0.1", num_shard=8) as handle:
+        yield handle
+
+
+@pytest.fixture(scope="module")
+async def flash_mixtral(flash_mixtral_handle):
+    await flash_mixtral_handle.health(300)
+    return flash_mixtral_handle.client
+
+
+@pytest.mark.skip(reason="requires > 4 shards")
+@pytest.mark.asyncio
+async def test_flash_mixtral(flash_mixtral, response_snapshot):
+    response = await flash_mixtral.generate(
+        "What is gradient descent?\n\n", max_new_tokens=10, decoder_input_details=True
+    )
+
+    assert response.details.generated_tokens == 10
+    assert (
+        response.generated_text
+        == "Gradient descent is an optimization algorithm used to minimize"
+    )
+    assert response == response_snapshot
+
+
+@pytest.mark.skip(reason="requires > 4 shards")
+@pytest.mark.asyncio
+async def test_flash_mixtral_all_params(flash_mixtral, response_snapshot):
+    response = await flash_mixtral.generate(
+        "What is gradient descent?\n\n",
+        max_new_tokens=10,
+        repetition_penalty=1.2,
+        return_full_text=True,
+        stop_sequences=["test"],
+        temperature=0.5,
+        top_p=0.9,
+        top_k=10,
+        truncate=5,
+        typical_p=0.9,
+        watermark=True,
+        decoder_input_details=True,
+        seed=0,
+    )
+
+    assert response.details.generated_tokens == 10
+    assert (
+        response.generated_text
+        == "What is gradient descent?\n\nIt seems to me, that if you're"
+    )
+    assert response == response_snapshot
+
+
+@pytest.mark.skip(reason="requires > 4 shards")
+@pytest.mark.asyncio
+async def test_flash_mixtral_load(flash_mixtral, generate_load, response_snapshot):
+    responses = await generate_load(
+        flash_mixtral, "What is gradient descent?\n\n", max_new_tokens=10, n=4
+    )
+
+    assert len(responses) == 4
+    assert responses[0].details.generated_tokens == 10
+    assert (
+        responses[0].generated_text
+        == "Gradient descent is an optimization algorithm used to minimize"
+    )
+    assert all(
+        [r.generated_text == responses[0].generated_text for r in responses]
+    ), f"{[r.generated_text  for r in responses]}"
+
+    assert responses == response_snapshot
diff --git a/integration-tests/models/test_flash_mixtral_gptq.py b/integration-tests/models/test_flash_mixtral_gptq.py
new file mode 100644
index 00000000..eb880628
--- /dev/null
+++ b/integration-tests/models/test_flash_mixtral_gptq.py
@@ -0,0 +1,60 @@
+import pytest
+
+
+@pytest.fixture(scope="module")
+def flash_mixtral_gptq_handle(launcher):
+    with launcher("TheBloke/Mixtral-8x7B-Instruct-v0.1-GPTQ", num_shard=2) as handle:
+        yield handle
+
+
+@pytest.fixture(scope="module")
+async def flash_mixtral_gptq(flash_mixtral_gptq_handle):
+    await flash_mixtral_gptq_handle.health(300)
+    return flash_mixtral_gptq_handle.client
+
+
+@pytest.mark.asyncio
+async def test_flash_mixtral_gptq(flash_mixtral_gptq, response_snapshot):
+    response = await flash_mixtral_gptq.generate(
+        "Test request", max_new_tokens=10, decoder_input_details=True
+    )
+
+    assert response == response_snapshot
+
+
+@pytest.mark.asyncio
+async def test_flash_mixtral_gptq_all_params(flash_mixtral_gptq, response_snapshot):
+    response = await flash_mixtral_gptq.generate(
+        "Test request",
+        max_new_tokens=10,
+        repetition_penalty=1.2,
+        return_full_text=True,
+        stop_sequences=["test"],
+        temperature=0.5,
+        top_p=0.9,
+        top_k=10,
+        truncate=5,
+        typical_p=0.9,
+        watermark=True,
+        decoder_input_details=True,
+        seed=0,
+    )
+
+    assert response.details.generated_tokens == 10
+    assert response == response_snapshot
+
+
+@pytest.mark.asyncio
+async def test_flash_mixtral_gptq_load(
+    flash_mixtral_gptq, generate_load, response_snapshot
+):
+    responses = await generate_load(
+        flash_mixtral_gptq, "Test request", max_new_tokens=10, n=4
+    )
+
+    assert len(responses) == 4
+    assert all(
+        [r.generated_text == responses[0].generated_text for r in responses]
+    ), f"{[r.generated_text  for r in responses]}"
+
+    assert responses == response_snapshot
diff --git a/integration-tests/models/test_flash_neox.py b/integration-tests/models/test_flash_neox.py
index 0289c61d..31848dae 100644
--- a/integration-tests/models/test_flash_neox.py
+++ b/integration-tests/models/test_flash_neox.py
@@ -13,6 +13,7 @@ async def flash_neox(flash_neox_handle):
     return flash_neox_handle.client
 
 
+@pytest.mark.release
 @pytest.mark.skip
 @pytest.mark.asyncio
 async def test_flash_neox(flash_neox, response_snapshot):
@@ -26,6 +27,7 @@ async def test_flash_neox(flash_neox, response_snapshot):
     assert response == response_snapshot
 
 
+@pytest.mark.release
 @pytest.mark.skip
 @pytest.mark.asyncio
 async def test_flash_neox_load(flash_neox, generate_load, response_snapshot):
diff --git a/integration-tests/models/test_flash_neox_sharded.py b/integration-tests/models/test_flash_neox_sharded.py
index 8a491915..1f1e7225 100644
--- a/integration-tests/models/test_flash_neox_sharded.py
+++ b/integration-tests/models/test_flash_neox_sharded.py
@@ -13,6 +13,7 @@ async def flash_neox_sharded(flash_neox_sharded_handle):
     return flash_neox_sharded_handle.client
 
 
+@pytest.mark.release
 @pytest.mark.asyncio
 async def test_flash_neox(flash_neox_sharded, response_snapshot):
     response = await flash_neox_sharded.generate(
@@ -25,6 +26,7 @@ async def test_flash_neox(flash_neox_sharded, response_snapshot):
     assert response == response_snapshot
 
 
+@pytest.mark.release
 @pytest.mark.asyncio
 async def test_flash_neox_load(flash_neox_sharded, generate_load, response_snapshot):
     responses = await generate_load(
diff --git a/integration-tests/models/test_flash_pali_gemma.py b/integration-tests/models/test_flash_pali_gemma.py
index d4e83c9f..52ecaed4 100644
--- a/integration-tests/models/test_flash_pali_gemma.py
+++ b/integration-tests/models/test_flash_pali_gemma.py
@@ -1,6 +1,4 @@
 import pytest
-import requests
-import io
 import base64
 
 
@@ -22,12 +20,19 @@ async def flash_pali_gemma(flash_pali_gemma_handle):
     return flash_pali_gemma_handle.client
 
 
+def get_chicken():
+    with open("integration-tests/images/chicken_on_money.png", "rb") as image_file:
+        encoded_string = base64.b64encode(image_file.read())
+    return f"data:image/png;base64,{encoded_string.decode('utf-8')}"
+
+
 def get_cow_beach():
     with open("integration-tests/images/cow_beach.png", "rb") as image_file:
         encoded_string = base64.b64encode(image_file.read())
     return f"data:image/png;base64,{encoded_string.decode('utf-8')}"
 
 
+@pytest.mark.release
 @pytest.mark.asyncio
 @pytest.mark.private
 async def test_flash_pali_gemma(flash_pali_gemma, response_snapshot):
@@ -37,3 +42,21 @@ async def test_flash_pali_gemma(flash_pali_gemma, response_snapshot):
 
     assert response.generated_text == "beach"
     assert response == response_snapshot
+
+
+@pytest.mark.release
+@pytest.mark.asyncio
+@pytest.mark.private
+async def test_flash_pali_gemma_two_images(flash_pali_gemma, response_snapshot):
+    chicken = get_chicken()
+    cow_beach = get_cow_beach()
+    response = await flash_pali_gemma.generate(
+        f"caption![]({chicken})![]({cow_beach})\n",
+        max_new_tokens=20,
+    )
+    # Is PaliGemma not able to handle two separate images? At least we
+    # get output showing that both images are used.
+    assert (
+        response.generated_text == "image result for chicken on the beach"
+    ), f"{repr(response.generated_text)}"
+    assert response == response_snapshot
diff --git a/integration-tests/models/test_flash_phi.py b/integration-tests/models/test_flash_phi.py
index 9d6ca566..73bb5edc 100644
--- a/integration-tests/models/test_flash_phi.py
+++ b/integration-tests/models/test_flash_phi.py
@@ -13,6 +13,7 @@ async def flash_phi(flash_phi_handle):
     return flash_phi_handle.client
 
 
+@pytest.mark.release
 @pytest.mark.asyncio
 async def test_flash_phi(flash_phi, response_snapshot):
     response = await flash_phi.generate(
@@ -24,6 +25,7 @@ async def test_flash_phi(flash_phi, response_snapshot):
     assert response == response_snapshot
 
 
+@pytest.mark.release
 @pytest.mark.asyncio
 async def test_flash_phi_all_params(flash_phi, response_snapshot):
     response = await flash_phi.generate(
@@ -47,6 +49,7 @@ async def test_flash_phi_all_params(flash_phi, response_snapshot):
     assert response == response_snapshot
 
 
+@pytest.mark.release
 @pytest.mark.asyncio
 async def test_flash_phi_load(flash_phi, generate_load, response_snapshot):
     responses = await generate_load(flash_phi, "Test request", max_new_tokens=10, n=4)
diff --git a/integration-tests/models/test_flash_phi35_moe.py b/integration-tests/models/test_flash_phi35_moe.py
new file mode 100644
index 00000000..2173740a
--- /dev/null
+++ b/integration-tests/models/test_flash_phi35_moe.py
@@ -0,0 +1,75 @@
+import pytest
+
+
+@pytest.fixture(scope="module")
+def flash_phi35_moe_handle(launcher):
+    with launcher(
+        "microsoft/Phi-3.5-MoE-instruct",
+        num_shard=4,
+    ) as handle:
+        yield handle
+
+
+@pytest.fixture(scope="module")
+async def flash_phi35_moe(flash_phi35_moe_handle):
+    await flash_phi35_moe_handle.health(300)
+    return flash_phi35_moe_handle.client
+
+
+@pytest.mark.asyncio
+async def test_flash_phi35_moe(flash_phi35_moe, response_snapshot):
+    response = await flash_phi35_moe.generate(
+        "What is gradient descent?\n\n", max_new_tokens=10, decoder_input_details=True
+    )
+
+    assert response.details.generated_tokens == 10
+    assert (
+        response.generated_text
+        == "Gradient descent is a first-order optimization algorithm"
+    )
+    assert response == response_snapshot
+
+
+@pytest.mark.asyncio
+async def test_flash_phi35_moe_all_params(flash_phi35_moe, response_snapshot):
+    response = await flash_phi35_moe.generate(
+        "What is gradient descent?\n\n",
+        max_new_tokens=10,
+        repetition_penalty=1.2,
+        return_full_text=True,
+        stop_sequences=["test"],
+        temperature=0.5,
+        top_p=0.9,
+        top_k=10,
+        truncate=5,
+        typical_p=0.9,
+        watermark=True,
+        decoder_input_details=True,
+        seed=0,
+    )
+
+    assert response.details.generated_tokens == 10
+    assert (
+        response.generated_text
+        == "What is gradient descent?\n\nHello! It seems you're addressing a"
+    )
+    assert response == response_snapshot
+
+
+@pytest.mark.asyncio
+async def test_flash_phi35_moe_load(flash_phi35_moe, generate_load, response_snapshot):
+    responses = await generate_load(
+        flash_phi35_moe, "What is gradient descent?\n\n", max_new_tokens=10, n=4
+    )
+
+    assert len(responses) == 4
+    assert responses[0].details.generated_tokens == 10
+    assert (
+        responses[0].generated_text
+        == "Gradient descent is a first-order optimization algorithm"
+    )
+    assert all(
+        [r.generated_text == responses[0].generated_text for r in responses]
+    ), f"{[r.generated_text  for r in responses]}"
+
+    assert responses == response_snapshot
diff --git a/integration-tests/models/test_flash_qwen2.py b/integration-tests/models/test_flash_qwen2.py
index 2963aeb4..c64f8732 100644
--- a/integration-tests/models/test_flash_qwen2.py
+++ b/integration-tests/models/test_flash_qwen2.py
@@ -13,6 +13,7 @@ async def flash_qwen2(flash_qwen2_handle):
     return flash_qwen2_handle.client
 
 
+@pytest.mark.release
 @pytest.mark.asyncio
 async def test_flash_qwen2(flash_qwen2, response_snapshot):
     response = await flash_qwen2.generate(
@@ -24,6 +25,7 @@ async def test_flash_qwen2(flash_qwen2, response_snapshot):
     assert response == response_snapshot
 
 
+@pytest.mark.release
 @pytest.mark.asyncio
 async def test_flash_qwen2_all_params(flash_qwen2, response_snapshot):
     response = await flash_qwen2.generate(
@@ -46,6 +48,7 @@ async def test_flash_qwen2_all_params(flash_qwen2, response_snapshot):
     assert response == response_snapshot
 
 
+@pytest.mark.release
 @pytest.mark.asyncio
 async def test_flash_qwen2_load(flash_qwen2, generate_load, response_snapshot):
     responses = await generate_load(flash_qwen2, "Test request", max_new_tokens=10, n=4)
diff --git a/integration-tests/models/test_flash_santacoder.py b/integration-tests/models/test_flash_santacoder.py
index 0f005f15..96a36aba 100644
--- a/integration-tests/models/test_flash_santacoder.py
+++ b/integration-tests/models/test_flash_santacoder.py
@@ -13,6 +13,7 @@ async def flash_santacoder(flash_santacoder_handle):
     return flash_santacoder_handle.client
 
 
+@pytest.mark.release
 @pytest.mark.asyncio
 async def test_flash_santacoder(flash_santacoder, response_snapshot):
     response = await flash_santacoder.generate(
@@ -23,6 +24,7 @@ async def test_flash_santacoder(flash_santacoder, response_snapshot):
     assert response == response_snapshot
 
 
+@pytest.mark.release
 @pytest.mark.asyncio
 async def test_flash_santacoder_load(
     flash_santacoder, generate_load, response_snapshot
diff --git a/integration-tests/models/test_flash_starcoder.py b/integration-tests/models/test_flash_starcoder.py
index 64e8b27c..dc5a8a53 100644
--- a/integration-tests/models/test_flash_starcoder.py
+++ b/integration-tests/models/test_flash_starcoder.py
@@ -13,6 +13,7 @@ async def flash_starcoder(flash_starcoder_handle):
     return flash_starcoder_handle.client
 
 
+@pytest.mark.release
 @pytest.mark.asyncio
 @pytest.mark.private
 async def test_flash_starcoder(flash_starcoder, response_snapshot):
@@ -24,6 +25,7 @@ async def test_flash_starcoder(flash_starcoder, response_snapshot):
     assert response == response_snapshot
 
 
+@pytest.mark.release
 @pytest.mark.asyncio
 @pytest.mark.private
 async def test_flash_starcoder_default_params(flash_starcoder, response_snapshot):
@@ -40,6 +42,7 @@ async def test_flash_starcoder_default_params(flash_starcoder, response_snapshot
     assert response == response_snapshot
 
 
+@pytest.mark.release
 @pytest.mark.asyncio
 @pytest.mark.private
 async def test_flash_starcoder_load(flash_starcoder, generate_load, response_snapshot):
diff --git a/integration-tests/models/test_flash_starcoder2.py b/integration-tests/models/test_flash_starcoder2.py
index ea665b6c..88341cfe 100644
--- a/integration-tests/models/test_flash_starcoder2.py
+++ b/integration-tests/models/test_flash_starcoder2.py
@@ -13,6 +13,7 @@ async def flash_starcoder2(flash_starcoder2_handle):
     return flash_starcoder2_handle.client
 
 
+@pytest.mark.release
 @pytest.mark.asyncio
 @pytest.mark.private
 async def test_flash_starcoder2(flash_starcoder2, response_snapshot):
@@ -24,6 +25,7 @@ async def test_flash_starcoder2(flash_starcoder2, response_snapshot):
     assert response == response_snapshot
 
 
+@pytest.mark.release
 @pytest.mark.asyncio
 @pytest.mark.private
 async def test_flash_starcoder2_default_params(flash_starcoder2, response_snapshot):
@@ -40,6 +42,7 @@ async def test_flash_starcoder2_default_params(flash_starcoder2, response_snapsh
     assert response == response_snapshot
 
 
+@pytest.mark.release
 @pytest.mark.asyncio
 @pytest.mark.private
 async def test_flash_starcoder2_load(
diff --git a/integration-tests/models/test_flash_starcoder_gptq.py b/integration-tests/models/test_flash_starcoder_gptq.py
index 329158b7..6d46e54d 100644
--- a/integration-tests/models/test_flash_starcoder_gptq.py
+++ b/integration-tests/models/test_flash_starcoder_gptq.py
@@ -13,6 +13,7 @@ async def flash_starcoder_gptq(flash_starcoder_gptq_handle):
     return flash_starcoder_gptq_handle.client
 
 
+@pytest.mark.release
 @pytest.mark.asyncio
 async def test_flash_starcoder_gptq(flash_starcoder_gptq, generous_response_snapshot):
     response = await flash_starcoder_gptq.generate(
@@ -20,10 +21,11 @@ async def test_flash_starcoder_gptq(flash_starcoder_gptq, generous_response_snap
         max_new_tokens=20,
         decoder_input_details=True,
     )
-    assert response.details.generated_tokens == 20
+    assert response.details.generated_tokens == 2
     assert response == generous_response_snapshot
 
 
+@pytest.mark.release
 @pytest.mark.asyncio
 async def test_flash_starcoder_gptq_default_params(
     flash_starcoder_gptq, generous_response_snapshot
@@ -36,10 +38,11 @@ async def test_flash_starcoder_gptq_default_params(
         decoder_input_details=True,
         seed=0,
     )
-    assert response.details.generated_tokens == 20
+    assert response.details.generated_tokens == 2
     assert response == generous_response_snapshot
 
 
+@pytest.mark.release
 @pytest.mark.asyncio
 async def test_flash_starcoder_gptq_load(
     flash_starcoder_gptq, generate_load, generous_response_snapshot
diff --git a/integration-tests/models/test_grammar_llama.py b/integration-tests/models/test_grammar_llama.py
index ce5da8a9..4face9e1 100644
--- a/integration-tests/models/test_grammar_llama.py
+++ b/integration-tests/models/test_grammar_llama.py
@@ -21,6 +21,7 @@ async def non_flash_llama_grammar(non_flash_llama_grammar_handle):
     return non_flash_llama_grammar_handle.client
 
 
+@pytest.mark.release
 @pytest.mark.skip
 @pytest.mark.asyncio
 async def test_non_flash_llama_grammar_json(non_flash_llama_grammar, response_snapshot):
diff --git a/integration-tests/models/test_grammar_response_format_llama.py b/integration-tests/models/test_grammar_response_format_llama.py
new file mode 100644
index 00000000..25bf9d98
--- /dev/null
+++ b/integration-tests/models/test_grammar_response_format_llama.py
@@ -0,0 +1,103 @@
+import pytest
+import requests
+from pydantic import BaseModel
+from typing import List
+
+
+@pytest.fixture(scope="module")
+def llama_grammar_handle(launcher):
+    with launcher(
+        "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
+        num_shard=1,
+        disable_grammar_support=False,
+        use_flash_attention=False,
+        max_batch_prefill_tokens=3000,
+    ) as handle:
+        yield handle
+
+
+@pytest.fixture(scope="module")
+async def llama_grammar(llama_grammar_handle):
+    await llama_grammar_handle.health(300)
+    return llama_grammar_handle.client
+
+
+@pytest.mark.release
+@pytest.mark.asyncio
+async def test_grammar_response_format_llama_json(llama_grammar, response_snapshot):
+
+    class Weather(BaseModel):
+        unit: str
+        temperature: List[int]
+
+    # send the request
+    response = requests.post(
+        f"{llama_grammar.base_url}/v1/chat/completions",
+        headers=llama_grammar.headers,
+        json={
+            "model": "tgi",
+            "messages": [
+                {
+                    "role": "system",
+                    "content": f"Respond to the users questions and answer them in the following format: {Weather.schema()}",
+                },
+                {
+                    "role": "user",
+                    "content": "What's the weather like the next 3 days in San Francisco, CA?",
+                },
+            ],
+            "seed": 42,
+            "max_tokens": 500,
+            "response_format": {"type": "json_object", "value": Weather.schema()},
+        },
+    )
+
+    chat_completion = response.json()
+    called = chat_completion["choices"][0]["message"]["content"]
+
+    assert response.status_code == 200
+    assert (
+        called
+        == '{\n  "temperature": [\n    35,\n    34,\n    36\n  ],\n  "unit": "°c"\n}'
+    )
+    assert chat_completion == response_snapshot
+
+
+@pytest.mark.release
+@pytest.mark.asyncio
+async def test_grammar_response_format_llama_error_if_tools_not_installed(
+    llama_grammar,
+):
+    class Weather(BaseModel):
+        unit: str
+        temperature: List[int]
+
+    # send the request
+    response = requests.post(
+        f"{llama_grammar.base_url}/v1/chat/completions",
+        headers=llama_grammar.headers,
+        json={
+            "model": "tgi",
+            "messages": [
+                {
+                    "role": "system",
+                    "content": f"Respond to the users questions and answer them in the following format: {Weather.schema()}",
+                },
+                {
+                    "role": "user",
+                    "content": "What's the weather like the next 3 days in San Francisco, CA?",
+                },
+            ],
+            "seed": 42,
+            "max_tokens": 500,
+            "tools": [],
+            "response_format": {"type": "json_object", "value": Weather.schema()},
+        },
+    )
+
+    # 422 means the server was unable to process the request because it contains invalid data.
+    assert response.status_code == 422
+    assert response.json() == {
+        "error": "Tool error: Grammar and tools are mutually exclusive",
+        "error_type": "tool_error",
+    }
diff --git a/integration-tests/models/test_idefics.py b/integration-tests/models/test_idefics.py
index aeeaffa1..eb573385 100644
--- a/integration-tests/models/test_idefics.py
+++ b/integration-tests/models/test_idefics.py
@@ -23,6 +23,12 @@ def get_chicken():
     return f"data:image/png;base64,{encoded_string.decode('utf-8')}"
 
 
+def get_cow_beach():
+    with open("integration-tests/images/cow_beach.png", "rb") as image_file:
+        encoded_string = base64.b64encode(image_file.read())
+    return f"data:image/png;base64,{encoded_string.decode('utf-8')}"
+
+
 @pytest.mark.asyncio
 async def test_idefics(idefics, response_snapshot):
     chicken = get_chicken()
@@ -39,6 +45,23 @@ async def test_idefics(idefics, response_snapshot):
     assert response == response_snapshot
 
 
+@pytest.mark.release
+@pytest.mark.asyncio
+@pytest.mark.private
+async def test_idefics_two_images(idefics, response_snapshot):
+    chicken = get_chicken()
+    cow_beach = get_cow_beach()
+    response = await idefics.generate(
+        f"User:![]({chicken})![]({cow_beach})Where are the cow and chicken?<end_of_utterance> \nAssistant:",
+        max_new_tokens=20,
+    )
+    assert (
+        response.generated_text == " The cow and chicken are on a beach."
+    ), f"{repr(response.generated_text)}"
+    assert response == response_snapshot
+
+
+@pytest.mark.release
 @pytest.mark.asyncio
 async def test_idefics_load(idefics, generate_load, response_snapshot):
     chicken = get_chicken()
@@ -51,9 +74,7 @@ async def test_idefics_load(idefics, generate_load, response_snapshot):
 
     generated_texts = [r.generated_text for r in responses]
 
-    assert (
-        generated_texts[0] == " \nAssistant: A rooster stands"
-    ), f"{response.generated_text}"
+    assert generated_texts[0] == " \nAssistant: A rooster stands"
     assert len(generated_texts) == 4
     assert generated_texts, all(
         [text == generated_texts[0] for text in generated_texts]
diff --git a/integration-tests/models/test_idefics2.py b/integration-tests/models/test_idefics2.py
index d34cce34..c5f48da3 100644
--- a/integration-tests/models/test_idefics2.py
+++ b/integration-tests/models/test_idefics2.py
@@ -9,6 +9,12 @@ def get_chicken():
     return f"data:image/png;base64,{encoded_string.decode('utf-8')}"
 
 
+def get_cow_beach():
+    with open("integration-tests/images/cow_beach.png", "rb") as image_file:
+        encoded_string = base64.b64encode(image_file.read())
+    return f"data:image/png;base64,{encoded_string.decode('utf-8')}"
+
+
 @pytest.fixture(scope="module")
 def flash_idefics2_next_handle(launcher):
     with launcher(
@@ -38,6 +44,23 @@ async def test_flash_idefics2_next_simple(flash_idefics2_next, response_snapshot
     assert response == response_snapshot
 
 
+@pytest.mark.asyncio
+@pytest.mark.private
+async def test_flash_idefics2_two_images(flash_idefics2_next, response_snapshot):
+    chicken = get_chicken()
+    cow_beach = get_cow_beach()
+    response = await flash_idefics2_next.generate(
+        f"User:![]({chicken})![]({cow_beach})Where are the cow and chicken?<end_of_utterance> \nAssistant:",
+        max_new_tokens=20,
+    )
+    assert (
+        response.generated_text
+        == " The cow is standing on the beach and the chicken is sitting on a pile of money."
+    ), f"{repr(response.generated_text)}"
+    assert response.details.generated_tokens == 19
+    assert response == response_snapshot
+
+
 @pytest.mark.asyncio
 @pytest.mark.private
 async def test_flash_idefics2_next_all_params(flash_idefics2_next, response_snapshot):
diff --git a/integration-tests/models/test_llava_next.py b/integration-tests/models/test_llava_next.py
index f5b290b1..ea277d71 100644
--- a/integration-tests/models/test_llava_next.py
+++ b/integration-tests/models/test_llava_next.py
@@ -26,6 +26,7 @@ async def flash_llava_next(flash_llava_next_handle):
     return flash_llava_next_handle.client
 
 
+@pytest.mark.release
 @pytest.mark.asyncio
 @pytest.mark.private
 async def test_flash_llava_next_simple(flash_llava_next, response_snapshot):
@@ -41,6 +42,7 @@ async def test_flash_llava_next_simple(flash_llava_next, response_snapshot):
     assert response == response_snapshot
 
 
+@pytest.mark.release
 @pytest.mark.asyncio
 @pytest.mark.private
 async def test_flash_llava_next_all_params(flash_llava_next, response_snapshot):
@@ -64,6 +66,7 @@ async def test_flash_llava_next_all_params(flash_llava_next, response_snapshot):
     assert response == response_snapshot
 
 
+@pytest.mark.release
 @pytest.mark.asyncio
 @pytest.mark.private
 async def test_flash_llava_next_load(
diff --git a/integration-tests/models/test_lora_mistral.py b/integration-tests/models/test_lora_mistral.py
new file mode 100644
index 00000000..ccdc1486
--- /dev/null
+++ b/integration-tests/models/test_lora_mistral.py
@@ -0,0 +1,134 @@
+import pytest
+import requests
+
+
+@pytest.fixture(scope="module")
+def lora_mistral_handle(launcher):
+    with launcher(
+        "mistralai/Mistral-7B-v0.1",
+        lora_adapters=[
+            "predibase/dbpedia",
+            "predibase/customer_support",
+        ],
+        cuda_graphs=[0],
+    ) as handle:
+        yield handle
+
+
+@pytest.fixture(scope="module")
+async def lora_mistral(lora_mistral_handle):
+    await lora_mistral_handle.health(300)
+    return lora_mistral_handle.client
+
+
+@pytest.mark.asyncio
+@pytest.mark.private
+async def test_lora_mistral(lora_mistral, response_snapshot):
+    response = await lora_mistral.generate(
+        "Test request", max_new_tokens=10, decoder_input_details=True
+    )
+    assert response.details.generated_tokens == 10
+
+
+classification_prompt = """You are given the title and the body of an article below. Please determine the type of the article.\n### Title: Great White Whale\n\n### Body: Great White Whale is the debut album by the Canadian rock band Secret and Whisper. The album was in the works for about a year and was released on February 12 2008. A music video was shot in Pittsburgh for the album's first single XOXOXO. The album reached number 17 on iTunes's top 100 albums in its first week on sale.\n\n### Article Type:"""
+
+
+@pytest.mark.asyncio
+@pytest.mark.private
+async def test_lora_mistral_without_adapter(lora_mistral, response_snapshot):
+    response = requests.post(
+        f"{lora_mistral.base_url}/generate",
+        headers=lora_mistral.headers,
+        json={
+            "inputs": classification_prompt,
+            "parameters": {
+                "max_new_tokens": 40,
+                "details": True,
+            },
+        },
+    )
+
+    assert response.status_code == 200
+    data = response.json()
+    assert (
+        data["generated_text"]
+        == "\n\n### 1. News\n### 2. Blog\n### 3. Article\n### 4. Review\n### 5. Other\n\n\n\n\n\n\n\n\n"
+    )
+    assert data == response_snapshot
+
+
+@pytest.mark.asyncio
+@pytest.mark.private
+async def test_lora_mistral_with_dbpedia_adapter(lora_mistral, response_snapshot):
+    response = requests.post(
+        f"{lora_mistral.base_url}/generate",
+        headers=lora_mistral.headers,
+        json={
+            "inputs": classification_prompt,
+            "parameters": {
+                "max_new_tokens": 40,
+                "adapter_id": "predibase/dbpedia",
+                "details": True,
+            },
+        },
+    )
+
+    assert response.status_code == 200
+    data = response.json()
+    assert data["generated_text"] == "  11"
+    assert data == response_snapshot
+
+
+@pytest.mark.asyncio
+@pytest.mark.private
+async def test_lora_mistral_with_customer_support_adapter(
+    lora_mistral, response_snapshot
+):
+    print(lora_mistral.base_url)
+    print(lora_mistral.headers)
+    response = requests.post(
+        f"{lora_mistral.base_url}/generate",
+        headers=lora_mistral.headers,
+        json={
+            "inputs": "What are 3 unique words that describe you?",
+            "parameters": {
+                "max_new_tokens": 40,
+                "adapter_id": "predibase/customer_support",
+                "details": True,
+            },
+        },
+    )
+
+    assert response.status_code == 200
+    data = response.json()
+    assert (
+        data["generated_text"]
+        == "\n\nI’m not sure if I can come up with 3 unique words that describe me, but I’ll try.\n\n1. Creative\n2. Funny\n3."
+    )
+    assert data == response_snapshot
+
+
+@pytest.mark.asyncio
+@pytest.mark.private
+async def test_lora_mistral_without_customer_support_adapter(
+    lora_mistral, response_snapshot
+):
+    response = requests.post(
+        f"{lora_mistral.base_url}/generate",
+        headers=lora_mistral.headers,
+        json={
+            "inputs": "What are 3 unique words that describe you?",
+            "parameters": {
+                "max_new_tokens": 40,
+                "details": True,
+            },
+        },
+    )
+
+    assert response.status_code == 200
+    data = response.json()
+    assert (
+        data["generated_text"]
+        == "\n\nI’m a very passionate person. I’m very driven. I’m very determined.\n\nWhat is your favorite thing about being a teacher?\n\nI love the fact"
+    )
+    assert data == response_snapshot
diff --git a/integration-tests/models/test_mamba.py b/integration-tests/models/test_mamba.py
index bf3701b4..85ed8fd1 100644
--- a/integration-tests/models/test_mamba.py
+++ b/integration-tests/models/test_mamba.py
@@ -13,6 +13,7 @@ async def fused_kernel_mamba(fused_kernel_mamba_handle):
     return fused_kernel_mamba_handle.client
 
 
+@pytest.mark.release
 @pytest.mark.asyncio
 async def test_mamba(fused_kernel_mamba, response_snapshot):
     response = await fused_kernel_mamba.generate(
@@ -24,6 +25,7 @@ async def test_mamba(fused_kernel_mamba, response_snapshot):
     assert response == response_snapshot
 
 
+@pytest.mark.release
 @pytest.mark.asyncio
 async def test_mamba_all_params(fused_kernel_mamba, response_snapshot):
     response = await fused_kernel_mamba.generate(
@@ -50,6 +52,7 @@ async def test_mamba_all_params(fused_kernel_mamba, response_snapshot):
     assert response == response_snapshot
 
 
+@pytest.mark.release
 @pytest.mark.asyncio
 async def test_mamba_load(
     fused_kernel_mamba, generate_load, generous_response_snapshot
@@ -59,6 +62,7 @@ async def test_mamba_load(
     )
 
     assert len(responses) == 4
+    assert responses[0].generated_text == "\n\nDeep learning is a new type of machine"
     assert all([r.generated_text == responses[0].generated_text for r in responses])
     assert responses[0].generated_text == "\n\nDeep learning is a new type of machine"
 
diff --git a/integration-tests/models/test_mllama.py b/integration-tests/models/test_mllama.py
new file mode 100644
index 00000000..1b4264aa
--- /dev/null
+++ b/integration-tests/models/test_mllama.py
@@ -0,0 +1,105 @@
+import pytest
+import base64
+import asyncio
+
+
+@pytest.fixture(scope="module")
+def mllama_handle(launcher):
+    with launcher("meta-llama/Llama-3.2-11B-Vision-Instruct", num_shard=2) as handle:
+        yield handle
+
+
+@pytest.fixture(scope="module")
+async def mllama(mllama_handle):
+    await mllama_handle.health(300)
+    return mllama_handle.client
+
+
+# TODO fix the server parsser to count inline image tokens correctly
+def get_chicken():
+    with open("integration-tests/images/chicken_on_money.png", "rb") as image_file:
+        encoded_string = base64.b64encode(image_file.read())
+    return f"data:image/png;base64,{encoded_string.decode('utf-8')}"
+
+
+def get_cow_beach():
+    with open("integration-tests/images/cow_beach.png", "rb") as image_file:
+        encoded_string = base64.b64encode(image_file.read())
+    return f"data:image/png;base64,{encoded_string.decode('utf-8')}"
+
+
+@pytest.mark.asyncio
+async def test_mllama_simpl(mllama, response_snapshot):
+    # chicken = get_chicken()
+    response = await mllama.chat(
+        max_tokens=10,
+        temperature=0.0,
+        messages=[
+            {
+                "role": "user",
+                "content": [
+                    {
+                        "type": "text",
+                        "text": "Can you tell me a very short story based on the image?",
+                    },
+                    {
+                        "type": "image_url",
+                        "image_url": {
+                            "url": "https://raw.githubusercontent.com/huggingface/text-generation-inference/main/integration-tests/images/chicken_on_money.png"
+                        },
+                    },
+                ],
+            },
+        ],
+    )
+
+    assert response.usage == {
+        "completion_tokens": 10,
+        "prompt_tokens": 50,
+        "total_tokens": 60,
+    }
+    assert (
+        response.choices[0].message.content
+        == "In a bustling city, a chicken named Cluck"
+    )
+    assert response == response_snapshot
+
+
+@pytest.mark.release
+@pytest.mark.asyncio
+async def test_mllama_load(mllama, generate_load, response_snapshot):
+    futures = [
+        mllama.chat(
+            max_tokens=10,
+            temperature=0.0,
+            messages=[
+                {
+                    "role": "user",
+                    "content": [
+                        {
+                            "type": "text",
+                            "text": "Can you tell me a very short story based on the image?",
+                        },
+                        {
+                            "type": "image_url",
+                            "image_url": {
+                                "url": "https://raw.githubusercontent.com/huggingface/text-generation-inference/main/integration-tests/images/chicken_on_money.png"
+                            },
+                        },
+                    ],
+                },
+            ],
+        )
+        for i in range(4)
+    ]
+    responses = await asyncio.gather(*futures)
+
+    generated_texts = [response.choices[0].message.content for response in responses]
+
+    assert generated_texts[0] == "In a bustling city, a chicken named Cluck"
+    assert len(generated_texts) == 4
+    assert generated_texts, all(
+        [text == generated_texts[0] for text in generated_texts]
+    )
+
+    assert responses == response_snapshot
diff --git a/integration-tests/models/test_mpt.py b/integration-tests/models/test_mpt.py
index d58a8c5a..1832910a 100644
--- a/integration-tests/models/test_mpt.py
+++ b/integration-tests/models/test_mpt.py
@@ -13,6 +13,7 @@ async def mpt_sharded(mpt_sharded_handle):
     return mpt_sharded_handle.client
 
 
+@pytest.mark.release
 @pytest.mark.asyncio
 async def test_mpt(mpt_sharded, response_snapshot):
     response = await mpt_sharded.generate(
@@ -29,6 +30,7 @@ async def test_mpt(mpt_sharded, response_snapshot):
     assert response == response_snapshot
 
 
+@pytest.mark.release
 @pytest.mark.asyncio
 async def test_mpt_load(mpt_sharded, generate_load, response_snapshot):
     responses = await generate_load(
diff --git a/integration-tests/models/test_mt0_base.py b/integration-tests/models/test_mt0_base.py
index c877056a..e53d8ed4 100644
--- a/integration-tests/models/test_mt0_base.py
+++ b/integration-tests/models/test_mt0_base.py
@@ -13,6 +13,7 @@ async def mt0_base(mt0_base_handle):
     return mt0_base_handle.client
 
 
+@pytest.mark.release
 @pytest.mark.asyncio
 async def test_mt0_base(mt0_base, response_snapshot):
     response = await mt0_base.generate(
@@ -27,6 +28,7 @@ async def test_mt0_base(mt0_base, response_snapshot):
     assert response == response_snapshot
 
 
+@pytest.mark.release
 @pytest.mark.asyncio
 async def test_mt0_base_all_params(mt0_base, response_snapshot):
     response = await mt0_base.generate(
@@ -49,6 +51,7 @@ async def test_mt0_base_all_params(mt0_base, response_snapshot):
     assert response == response_snapshot
 
 
+@pytest.mark.release
 @pytest.mark.asyncio
 async def test_mt0_base_load(mt0_base, generate_load, response_snapshot):
     responses = await generate_load(
diff --git a/integration-tests/models/test_neox.py b/integration-tests/models/test_neox.py
index 7b88f86a..ee60441d 100644
--- a/integration-tests/models/test_neox.py
+++ b/integration-tests/models/test_neox.py
@@ -15,6 +15,7 @@ async def neox(neox_handle):
     return neox_handle.client
 
 
+@pytest.mark.release
 @pytest.mark.skip
 @pytest.mark.asyncio
 async def test_neox(neox, response_snapshot):
@@ -28,6 +29,7 @@ async def test_neox(neox, response_snapshot):
     assert response == response_snapshot
 
 
+@pytest.mark.release
 @pytest.mark.skip
 @pytest.mark.asyncio
 async def test_neox_load(neox, generate_load, response_snapshot):
diff --git a/integration-tests/models/test_neox_sharded.py b/integration-tests/models/test_neox_sharded.py
index 8cee8765..a69227c9 100644
--- a/integration-tests/models/test_neox_sharded.py
+++ b/integration-tests/models/test_neox_sharded.py
@@ -15,6 +15,7 @@ async def neox_sharded(neox_sharded_handle):
     return neox_sharded_handle.client
 
 
+@pytest.mark.release
 @pytest.mark.skip
 @pytest.mark.asyncio
 async def test_neox(neox_sharded, response_snapshot):
@@ -28,6 +29,7 @@ async def test_neox(neox_sharded, response_snapshot):
     assert response == response_snapshot
 
 
+@pytest.mark.release
 @pytest.mark.skip
 @pytest.mark.asyncio
 async def test_neox_load(neox_sharded, generate_load, response_snapshot):
diff --git a/integration-tests/models/test_opt.py b/integration-tests/models/test_opt.py
new file mode 100644
index 00000000..cbeb6376
--- /dev/null
+++ b/integration-tests/models/test_opt.py
@@ -0,0 +1,19 @@
+import pytest
+
+
+@pytest.fixture(scope="module")
+def opt_sharded_handle(launcher):
+    with launcher("facebook/opt-6.7b", num_shard=2) as handle:
+        yield handle
+
+
+@pytest.fixture(scope="module")
+async def opt_sharded(opt_sharded_handle):
+    await opt_sharded_handle.health(300)
+    return opt_sharded_handle.client
+
+
+@pytest.mark.release
+@pytest.mark.asyncio
+async def test_opt(opt_sharded):
+    pass
diff --git a/integration-tests/models/test_t5_sharded.py b/integration-tests/models/test_t5_sharded.py
index 4b4cfd98..24003024 100644
--- a/integration-tests/models/test_t5_sharded.py
+++ b/integration-tests/models/test_t5_sharded.py
@@ -13,6 +13,7 @@ async def t5_sharded(t5_sharded_handle):
     return t5_sharded_handle.client
 
 
+@pytest.mark.release
 @pytest.mark.asyncio
 async def test_t5_sharded(t5_sharded, response_snapshot):
     response = await t5_sharded.generate(
@@ -24,6 +25,7 @@ async def test_t5_sharded(t5_sharded, response_snapshot):
     assert response == response_snapshot
 
 
+@pytest.mark.release
 @pytest.mark.asyncio
 async def test_t5_sharded_load(t5_sharded, generate_load, response_snapshot):
     responses = await generate_load(
diff --git a/integration-tests/models/test_tools_llama.py b/integration-tests/models/test_tools_llama.py
index 0af3f66a..3c222b72 100644
--- a/integration-tests/models/test_tools_llama.py
+++ b/integration-tests/models/test_tools_llama.py
@@ -1,13 +1,12 @@
 import pytest
-import json
-
-from text_generation.types import GrammarType
 
 
 @pytest.fixture(scope="module")
 def flash_llama_grammar_tools_handle(launcher):
     with launcher(
-        "TinyLlama/TinyLlama-1.1B-Chat-v1.0", num_shard=2, disable_grammar_support=False
+        "meta-llama/Meta-Llama-3.1-8B-Instruct",
+        num_shard=2,
+        disable_grammar_support=False,
     ) as handle:
         yield handle
 
@@ -39,6 +38,7 @@ tools = [
                     },
                 },
                 "required": ["location", "format"],
+                "additionalProperties": False,
             },
         },
     },
@@ -65,13 +65,13 @@ tools = [
                     },
                 },
                 "required": ["location", "format", "num_days"],
+                "additionalProperties": False,
             },
         },
     },
 ]
 
 
-@pytest.mark.skip(reason="Takes too long to run")
 @pytest.mark.asyncio
 @pytest.mark.private
 async def test_flash_llama_grammar_tools(flash_llama_grammar_tools, response_snapshot):
@@ -79,7 +79,7 @@ async def test_flash_llama_grammar_tools(flash_llama_grammar_tools, response_sna
         max_tokens=100,
         seed=1,
         tools=tools,
-        presence_penalty=-1.1,
+        temperature=0.0,
         messages=[
             {
                 "role": "system",
@@ -91,22 +91,21 @@ async def test_flash_llama_grammar_tools(flash_llama_grammar_tools, response_sna
             },
         ],
     )
-    assert response.choices[0].message.content == None
+    assert response.choices[0].message.content is None
     assert response.choices[0].message.tool_calls == [
         {
-            "id": 0,
+            "id": "0",
             "type": "function",
             "function": {
                 "description": None,
                 "name": "get_current_weather",
-                "arguments": {"format": "celsius", "location": "New York, NY"},
+                "arguments": {"format": "celsius", "location": "Brooklyn, NY"},
             },
         }
     ]
     assert response == response_snapshot
 
 
-@pytest.mark.skip(reason="Takes too long to run")
 @pytest.mark.asyncio
 @pytest.mark.private
 async def test_flash_llama_grammar_tools_auto(
@@ -116,8 +115,8 @@ async def test_flash_llama_grammar_tools_auto(
         max_tokens=100,
         seed=1,
         tools=tools,
+        temperature=0.0,
         tool_choice="auto",
-        presence_penalty=-1.1,
         messages=[
             {
                 "role": "system",
@@ -129,15 +128,15 @@ async def test_flash_llama_grammar_tools_auto(
             },
         ],
     )
-    assert response.choices[0].message.content == None
+    assert response.choices[0].message.content is None
     assert response.choices[0].message.tool_calls == [
         {
-            "id": 0,
+            "id": "0",
             "type": "function",
             "function": {
                 "description": None,
                 "name": "get_current_weather",
-                "arguments": {"format": "celsius", "location": "New York, NY"},
+                "arguments": {"format": "celsius", "location": "Brooklyn, NY"},
             },
         }
     ]
@@ -145,7 +144,6 @@ async def test_flash_llama_grammar_tools_auto(
     assert response == response_snapshot
 
 
-@pytest.mark.skip(reason="Takes too long to run")
 @pytest.mark.asyncio
 @pytest.mark.private
 async def test_flash_llama_grammar_tools_choice(
@@ -155,8 +153,8 @@ async def test_flash_llama_grammar_tools_choice(
         max_tokens=100,
         seed=1,
         tools=tools,
+        temperature=0.0,
         tool_choice="get_current_weather",
-        presence_penalty=-1.1,
         messages=[
             {
                 "role": "system",
@@ -168,15 +166,15 @@ async def test_flash_llama_grammar_tools_choice(
             },
         ],
     )
-    assert response.choices[0].message.content == None
+    assert response.choices[0].message.content is None
     assert response.choices[0].message.tool_calls == [
         {
-            "id": 0,
+            "id": "0",
             "type": "function",
             "function": {
                 "description": None,
                 "name": "get_current_weather",
-                "arguments": {"format": "celsius", "location": "New York, NY"},
+                "arguments": {"format": "celsius", "location": "Brooklyn, NY"},
             },
         }
     ]
@@ -184,7 +182,6 @@ async def test_flash_llama_grammar_tools_choice(
     assert response == response_snapshot
 
 
-@pytest.mark.skip(reason="Takes too long to run")
 @pytest.mark.asyncio
 @pytest.mark.private
 async def test_flash_llama_grammar_tools_stream(
@@ -194,8 +191,8 @@ async def test_flash_llama_grammar_tools_stream(
         max_tokens=100,
         seed=1,
         tools=tools,
+        temperature=0.0,
         tool_choice="get_current_weather",
-        presence_penalty=-1.1,
         messages=[
             {
                 "role": "system",
@@ -213,11 +210,10 @@ async def test_flash_llama_grammar_tools_stream(
     async for response in responses:
         count += 1
 
-    assert count == 38
+    assert count == 28
     assert response == response_snapshot
 
 
-@pytest.mark.skip(reason="Takes too long to run")
 @pytest.mark.asyncio
 @pytest.mark.private
 async def test_flash_llama_grammar_tools_insufficient_information(
@@ -225,14 +221,10 @@ async def test_flash_llama_grammar_tools_insufficient_information(
 ):
     responses = await flash_llama_grammar_tools.chat(
         max_tokens=100,
-        seed=8,
+        seed=24,
         tools=tools,
         tool_choice="auto",
         messages=[
-            {
-                "role": "system",
-                "content": "ONLY RESPOND IF THE USER ASKS A WEATHER RELATED QUESTION",
-            },
             {
                 "role": "user",
                 "content": "Tell me a story about 3 sea creatures",
@@ -241,19 +233,5 @@ async def test_flash_llama_grammar_tools_insufficient_information(
         stream=False,
     )
 
-    assert responses.choices[0].message.content == None
-    assert responses.choices[0].message.tool_calls == [
-        {
-            "function": {
-                "arguments": {
-                    "error": "Cannot get current weather forecast from specified location and temperature unit. Please try again with different options."
-                },
-                "description": None,
-                "name": "notify_error",
-            },
-            "id": 0,
-            "type": "function",
-        }
-    ]
-
+    assert responses.choices[0].message.content == "There is a huge storm in the ocean"
     assert responses == response_snapshot
diff --git a/integration-tests/poetry.lock b/integration-tests/poetry.lock
index 3af99942..56b146bc 100644
--- a/integration-tests/poetry.lock
+++ b/integration-tests/poetry.lock
@@ -1,112 +1,127 @@
-# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand.
+# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand.
+
+[[package]]
+name = "aiohappyeyeballs"
+version = "2.4.0"
+description = "Happy Eyeballs for asyncio"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "aiohappyeyeballs-2.4.0-py3-none-any.whl", hash = "sha256:7ce92076e249169a13c2f49320d1967425eaf1f407522d707d59cac7628d62bd"},
+    {file = "aiohappyeyeballs-2.4.0.tar.gz", hash = "sha256:55a1714f084e63d49639800f95716da97a1f173d46a16dfcfda0016abb93b6b2"},
+]
 
 [[package]]
 name = "aiohttp"
-version = "3.8.5"
+version = "3.10.5"
 description = "Async http client/server framework (asyncio)"
 optional = false
-python-versions = ">=3.6"
+python-versions = ">=3.8"
 files = [
-    {file = "aiohttp-3.8.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a94159871304770da4dd371f4291b20cac04e8c94f11bdea1c3478e557fbe0d8"},
-    {file = "aiohttp-3.8.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:13bf85afc99ce6f9ee3567b04501f18f9f8dbbb2ea11ed1a2e079670403a7c84"},
-    {file = "aiohttp-3.8.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2ce2ac5708501afc4847221a521f7e4b245abf5178cf5ddae9d5b3856ddb2f3a"},
-    {file = "aiohttp-3.8.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:96943e5dcc37a6529d18766597c491798b7eb7a61d48878611298afc1fca946c"},
-    {file = "aiohttp-3.8.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2ad5c3c4590bb3cc28b4382f031f3783f25ec223557124c68754a2231d989e2b"},
-    {file = "aiohttp-3.8.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0c413c633d0512df4dc7fd2373ec06cc6a815b7b6d6c2f208ada7e9e93a5061d"},
-    {file = "aiohttp-3.8.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:df72ac063b97837a80d80dec8d54c241af059cc9bb42c4de68bd5b61ceb37caa"},
-    {file = "aiohttp-3.8.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c48c5c0271149cfe467c0ff8eb941279fd6e3f65c9a388c984e0e6cf57538e14"},
-    {file = "aiohttp-3.8.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:368a42363c4d70ab52c2c6420a57f190ed3dfaca6a1b19afda8165ee16416a82"},
-    {file = "aiohttp-3.8.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:7607ec3ce4993464368505888af5beb446845a014bc676d349efec0e05085905"},
-    {file = "aiohttp-3.8.5-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:0d21c684808288a98914e5aaf2a7c6a3179d4df11d249799c32d1808e79503b5"},
-    {file = "aiohttp-3.8.5-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:312fcfbacc7880a8da0ae8b6abc6cc7d752e9caa0051a53d217a650b25e9a691"},
-    {file = "aiohttp-3.8.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:ad093e823df03bb3fd37e7dec9d4670c34f9e24aeace76808fc20a507cace825"},
-    {file = "aiohttp-3.8.5-cp310-cp310-win32.whl", hash = "sha256:33279701c04351a2914e1100b62b2a7fdb9a25995c4a104259f9a5ead7ed4802"},
-    {file = "aiohttp-3.8.5-cp310-cp310-win_amd64.whl", hash = "sha256:6e4a280e4b975a2e7745573e3fc9c9ba0d1194a3738ce1cbaa80626cc9b4f4df"},
-    {file = "aiohttp-3.8.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ae871a964e1987a943d83d6709d20ec6103ca1eaf52f7e0d36ee1b5bebb8b9b9"},
-    {file = "aiohttp-3.8.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:461908b2578955045efde733719d62f2b649c404189a09a632d245b445c9c975"},
-    {file = "aiohttp-3.8.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:72a860c215e26192379f57cae5ab12b168b75db8271f111019509a1196dfc780"},
-    {file = "aiohttp-3.8.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cc14be025665dba6202b6a71cfcdb53210cc498e50068bc088076624471f8bb9"},
-    {file = "aiohttp-3.8.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8af740fc2711ad85f1a5c034a435782fbd5b5f8314c9a3ef071424a8158d7f6b"},
-    {file = "aiohttp-3.8.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:841cd8233cbd2111a0ef0a522ce016357c5e3aff8a8ce92bcfa14cef890d698f"},
-    {file = "aiohttp-3.8.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5ed1c46fb119f1b59304b5ec89f834f07124cd23ae5b74288e364477641060ff"},
-    {file = "aiohttp-3.8.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:84f8ae3e09a34f35c18fa57f015cc394bd1389bce02503fb30c394d04ee6b938"},
-    {file = "aiohttp-3.8.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:62360cb771707cb70a6fd114b9871d20d7dd2163a0feafe43fd115cfe4fe845e"},
-    {file = "aiohttp-3.8.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:23fb25a9f0a1ca1f24c0a371523546366bb642397c94ab45ad3aedf2941cec6a"},
-    {file = "aiohttp-3.8.5-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:b0ba0d15164eae3d878260d4c4df859bbdc6466e9e6689c344a13334f988bb53"},
-    {file = "aiohttp-3.8.5-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:5d20003b635fc6ae3f96d7260281dfaf1894fc3aa24d1888a9b2628e97c241e5"},
-    {file = "aiohttp-3.8.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0175d745d9e85c40dcc51c8f88c74bfbaef9e7afeeeb9d03c37977270303064c"},
-    {file = "aiohttp-3.8.5-cp311-cp311-win32.whl", hash = "sha256:2e1b1e51b0774408f091d268648e3d57f7260c1682e7d3a63cb00d22d71bb945"},
-    {file = "aiohttp-3.8.5-cp311-cp311-win_amd64.whl", hash = "sha256:043d2299f6dfdc92f0ac5e995dfc56668e1587cea7f9aa9d8a78a1b6554e5755"},
-    {file = "aiohttp-3.8.5-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:cae533195e8122584ec87531d6df000ad07737eaa3c81209e85c928854d2195c"},
-    {file = "aiohttp-3.8.5-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4f21e83f355643c345177a5d1d8079f9f28b5133bcd154193b799d380331d5d3"},
-    {file = "aiohttp-3.8.5-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a7a75ef35f2df54ad55dbf4b73fe1da96f370e51b10c91f08b19603c64004acc"},
-    {file = "aiohttp-3.8.5-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2e2e9839e14dd5308ee773c97115f1e0a1cb1d75cbeeee9f33824fa5144c7634"},
-    {file = "aiohttp-3.8.5-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c44e65da1de4403d0576473e2344828ef9c4c6244d65cf4b75549bb46d40b8dd"},
-    {file = "aiohttp-3.8.5-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:78d847e4cde6ecc19125ccbc9bfac4a7ab37c234dd88fbb3c5c524e8e14da543"},
-    {file = "aiohttp-3.8.5-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:c7a815258e5895d8900aec4454f38dca9aed71085f227537208057853f9d13f2"},
-    {file = "aiohttp-3.8.5-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:8b929b9bd7cd7c3939f8bcfffa92fae7480bd1aa425279d51a89327d600c704d"},
-    {file = "aiohttp-3.8.5-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:5db3a5b833764280ed7618393832e0853e40f3d3e9aa128ac0ba0f8278d08649"},
-    {file = "aiohttp-3.8.5-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:a0215ce6041d501f3155dc219712bc41252d0ab76474615b9700d63d4d9292af"},
-    {file = "aiohttp-3.8.5-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:fd1ed388ea7fbed22c4968dd64bab0198de60750a25fe8c0c9d4bef5abe13824"},
-    {file = "aiohttp-3.8.5-cp36-cp36m-win32.whl", hash = "sha256:6e6783bcc45f397fdebc118d772103d751b54cddf5b60fbcc958382d7dd64f3e"},
-    {file = "aiohttp-3.8.5-cp36-cp36m-win_amd64.whl", hash = "sha256:b5411d82cddd212644cf9360879eb5080f0d5f7d809d03262c50dad02f01421a"},
-    {file = "aiohttp-3.8.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:01d4c0c874aa4ddfb8098e85d10b5e875a70adc63db91f1ae65a4b04d3344cda"},
-    {file = "aiohttp-3.8.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e5980a746d547a6ba173fd5ee85ce9077e72d118758db05d229044b469d9029a"},
-    {file = "aiohttp-3.8.5-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2a482e6da906d5e6e653be079b29bc173a48e381600161c9932d89dfae5942ef"},
-    {file = "aiohttp-3.8.5-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:80bd372b8d0715c66c974cf57fe363621a02f359f1ec81cba97366948c7fc873"},
-    {file = "aiohttp-3.8.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c1161b345c0a444ebcf46bf0a740ba5dcf50612fd3d0528883fdc0eff578006a"},
-    {file = "aiohttp-3.8.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cd56db019015b6acfaaf92e1ac40eb8434847d9bf88b4be4efe5bfd260aee692"},
-    {file = "aiohttp-3.8.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:153c2549f6c004d2754cc60603d4668899c9895b8a89397444a9c4efa282aaf4"},
-    {file = "aiohttp-3.8.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:4a01951fabc4ce26ab791da5f3f24dca6d9a6f24121746eb19756416ff2d881b"},
-    {file = "aiohttp-3.8.5-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bfb9162dcf01f615462b995a516ba03e769de0789de1cadc0f916265c257e5d8"},
-    {file = "aiohttp-3.8.5-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:7dde0009408969a43b04c16cbbe252c4f5ef4574ac226bc8815cd7342d2028b6"},
-    {file = "aiohttp-3.8.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:4149d34c32f9638f38f544b3977a4c24052042affa895352d3636fa8bffd030a"},
-    {file = "aiohttp-3.8.5-cp37-cp37m-win32.whl", hash = "sha256:68c5a82c8779bdfc6367c967a4a1b2aa52cd3595388bf5961a62158ee8a59e22"},
-    {file = "aiohttp-3.8.5-cp37-cp37m-win_amd64.whl", hash = "sha256:2cf57fb50be5f52bda004b8893e63b48530ed9f0d6c96c84620dc92fe3cd9b9d"},
-    {file = "aiohttp-3.8.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:eca4bf3734c541dc4f374ad6010a68ff6c6748f00451707f39857f429ca36ced"},
-    {file = "aiohttp-3.8.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1274477e4c71ce8cfe6c1ec2f806d57c015ebf84d83373676036e256bc55d690"},
-    {file = "aiohttp-3.8.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:28c543e54710d6158fc6f439296c7865b29e0b616629767e685a7185fab4a6b9"},
-    {file = "aiohttp-3.8.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:910bec0c49637d213f5d9877105d26e0c4a4de2f8b1b29405ff37e9fc0ad52b8"},
-    {file = "aiohttp-3.8.5-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5443910d662db951b2e58eb70b0fbe6b6e2ae613477129a5805d0b66c54b6cb7"},
-    {file = "aiohttp-3.8.5-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2e460be6978fc24e3df83193dc0cc4de46c9909ed92dd47d349a452ef49325b7"},
-    {file = "aiohttp-3.8.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fb1558def481d84f03b45888473fc5a1f35747b5f334ef4e7a571bc0dfcb11f8"},
-    {file = "aiohttp-3.8.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:34dd0c107799dcbbf7d48b53be761a013c0adf5571bf50c4ecad5643fe9cfcd0"},
-    {file = "aiohttp-3.8.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:aa1990247f02a54185dc0dff92a6904521172a22664c863a03ff64c42f9b5410"},
-    {file = "aiohttp-3.8.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:0e584a10f204a617d71d359fe383406305a4b595b333721fa50b867b4a0a1548"},
-    {file = "aiohttp-3.8.5-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:a3cf433f127efa43fee6b90ea4c6edf6c4a17109d1d037d1a52abec84d8f2e42"},
-    {file = "aiohttp-3.8.5-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:c11f5b099adafb18e65c2c997d57108b5bbeaa9eeee64a84302c0978b1ec948b"},
-    {file = "aiohttp-3.8.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:84de26ddf621d7ac4c975dbea4c945860e08cccde492269db4e1538a6a6f3c35"},
-    {file = "aiohttp-3.8.5-cp38-cp38-win32.whl", hash = "sha256:ab88bafedc57dd0aab55fa728ea10c1911f7e4d8b43e1d838a1739f33712921c"},
-    {file = "aiohttp-3.8.5-cp38-cp38-win_amd64.whl", hash = "sha256:5798a9aad1879f626589f3df0f8b79b3608a92e9beab10e5fda02c8a2c60db2e"},
-    {file = "aiohttp-3.8.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:a6ce61195c6a19c785df04e71a4537e29eaa2c50fe745b732aa937c0c77169f3"},
-    {file = "aiohttp-3.8.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:773dd01706d4db536335fcfae6ea2440a70ceb03dd3e7378f3e815b03c97ab51"},
-    {file = "aiohttp-3.8.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f83a552443a526ea38d064588613aca983d0ee0038801bc93c0c916428310c28"},
-    {file = "aiohttp-3.8.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f7372f7341fcc16f57b2caded43e81ddd18df53320b6f9f042acad41f8e049a"},
-    {file = "aiohttp-3.8.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ea353162f249c8097ea63c2169dd1aa55de1e8fecbe63412a9bc50816e87b761"},
-    {file = "aiohttp-3.8.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e5d47ae48db0b2dcf70bc8a3bc72b3de86e2a590fc299fdbbb15af320d2659de"},
-    {file = "aiohttp-3.8.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d827176898a2b0b09694fbd1088c7a31836d1a505c243811c87ae53a3f6273c1"},
-    {file = "aiohttp-3.8.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3562b06567c06439d8b447037bb655ef69786c590b1de86c7ab81efe1c9c15d8"},
-    {file = "aiohttp-3.8.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:4e874cbf8caf8959d2adf572a78bba17cb0e9d7e51bb83d86a3697b686a0ab4d"},
-    {file = "aiohttp-3.8.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:6809a00deaf3810e38c628e9a33271892f815b853605a936e2e9e5129762356c"},
-    {file = "aiohttp-3.8.5-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:33776e945d89b29251b33a7e7d006ce86447b2cfd66db5e5ded4e5cd0340585c"},
-    {file = "aiohttp-3.8.5-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:eaeed7abfb5d64c539e2db173f63631455f1196c37d9d8d873fc316470dfbacd"},
-    {file = "aiohttp-3.8.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:e91d635961bec2d8f19dfeb41a539eb94bd073f075ca6dae6c8dc0ee89ad6f91"},
-    {file = "aiohttp-3.8.5-cp39-cp39-win32.whl", hash = "sha256:00ad4b6f185ec67f3e6562e8a1d2b69660be43070bd0ef6fcec5211154c7df67"},
-    {file = "aiohttp-3.8.5-cp39-cp39-win_amd64.whl", hash = "sha256:c0a9034379a37ae42dea7ac1e048352d96286626251862e448933c0f59cbd79c"},
-    {file = "aiohttp-3.8.5.tar.gz", hash = "sha256:b9552ec52cc147dbf1944ac7ac98af7602e51ea2dcd076ed194ca3c0d1c7d0bc"},
+    {file = "aiohttp-3.10.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:18a01eba2574fb9edd5f6e5fb25f66e6ce061da5dab5db75e13fe1558142e0a3"},
+    {file = "aiohttp-3.10.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:94fac7c6e77ccb1ca91e9eb4cb0ac0270b9fb9b289738654120ba8cebb1189c6"},
+    {file = "aiohttp-3.10.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2f1f1c75c395991ce9c94d3e4aa96e5c59c8356a15b1c9231e783865e2772699"},
+    {file = "aiohttp-3.10.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4f7acae3cf1a2a2361ec4c8e787eaaa86a94171d2417aae53c0cca6ca3118ff6"},
+    {file = "aiohttp-3.10.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:94c4381ffba9cc508b37d2e536b418d5ea9cfdc2848b9a7fea6aebad4ec6aac1"},
+    {file = "aiohttp-3.10.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c31ad0c0c507894e3eaa843415841995bf8de4d6b2d24c6e33099f4bc9fc0d4f"},
+    {file = "aiohttp-3.10.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0912b8a8fadeb32ff67a3ed44249448c20148397c1ed905d5dac185b4ca547bb"},
+    {file = "aiohttp-3.10.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0d93400c18596b7dc4794d48a63fb361b01a0d8eb39f28800dc900c8fbdaca91"},
+    {file = "aiohttp-3.10.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d00f3c5e0d764a5c9aa5a62d99728c56d455310bcc288a79cab10157b3af426f"},
+    {file = "aiohttp-3.10.5-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:d742c36ed44f2798c8d3f4bc511f479b9ceef2b93f348671184139e7d708042c"},
+    {file = "aiohttp-3.10.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:814375093edae5f1cb31e3407997cf3eacefb9010f96df10d64829362ae2df69"},
+    {file = "aiohttp-3.10.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:8224f98be68a84b19f48e0bdc14224b5a71339aff3a27df69989fa47d01296f3"},
+    {file = "aiohttp-3.10.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d9a487ef090aea982d748b1b0d74fe7c3950b109df967630a20584f9a99c0683"},
+    {file = "aiohttp-3.10.5-cp310-cp310-win32.whl", hash = "sha256:d9ef084e3dc690ad50137cc05831c52b6ca428096e6deb3c43e95827f531d5ef"},
+    {file = "aiohttp-3.10.5-cp310-cp310-win_amd64.whl", hash = "sha256:66bf9234e08fe561dccd62083bf67400bdbf1c67ba9efdc3dac03650e97c6088"},
+    {file = "aiohttp-3.10.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:8c6a4e5e40156d72a40241a25cc226051c0a8d816610097a8e8f517aeacd59a2"},
+    {file = "aiohttp-3.10.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2c634a3207a5445be65536d38c13791904fda0748b9eabf908d3fe86a52941cf"},
+    {file = "aiohttp-3.10.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4aff049b5e629ef9b3e9e617fa6e2dfeda1bf87e01bcfecaf3949af9e210105e"},
+    {file = "aiohttp-3.10.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1942244f00baaacaa8155eca94dbd9e8cc7017deb69b75ef67c78e89fdad3c77"},
+    {file = "aiohttp-3.10.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e04a1f2a65ad2f93aa20f9ff9f1b672bf912413e5547f60749fa2ef8a644e061"},
+    {file = "aiohttp-3.10.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7f2bfc0032a00405d4af2ba27f3c429e851d04fad1e5ceee4080a1c570476697"},
+    {file = "aiohttp-3.10.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:424ae21498790e12eb759040bbb504e5e280cab64693d14775c54269fd1d2bb7"},
+    {file = "aiohttp-3.10.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:975218eee0e6d24eb336d0328c768ebc5d617609affaca5dbbd6dd1984f16ed0"},
+    {file = "aiohttp-3.10.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4120d7fefa1e2d8fb6f650b11489710091788de554e2b6f8347c7a20ceb003f5"},
+    {file = "aiohttp-3.10.5-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:b90078989ef3fc45cf9221d3859acd1108af7560c52397ff4ace8ad7052a132e"},
+    {file = "aiohttp-3.10.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:ba5a8b74c2a8af7d862399cdedce1533642fa727def0b8c3e3e02fcb52dca1b1"},
+    {file = "aiohttp-3.10.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:02594361128f780eecc2a29939d9dfc870e17b45178a867bf61a11b2a4367277"},
+    {file = "aiohttp-3.10.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:8fb4fc029e135859f533025bc82047334e24b0d489e75513144f25408ecaf058"},
+    {file = "aiohttp-3.10.5-cp311-cp311-win32.whl", hash = "sha256:e1ca1ef5ba129718a8fc827b0867f6aa4e893c56eb00003b7367f8a733a9b072"},
+    {file = "aiohttp-3.10.5-cp311-cp311-win_amd64.whl", hash = "sha256:349ef8a73a7c5665cca65c88ab24abe75447e28aa3bc4c93ea5093474dfdf0ff"},
+    {file = "aiohttp-3.10.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:305be5ff2081fa1d283a76113b8df7a14c10d75602a38d9f012935df20731487"},
+    {file = "aiohttp-3.10.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3a1c32a19ee6bbde02f1cb189e13a71b321256cc1d431196a9f824050b160d5a"},
+    {file = "aiohttp-3.10.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:61645818edd40cc6f455b851277a21bf420ce347baa0b86eaa41d51ef58ba23d"},
+    {file = "aiohttp-3.10.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c225286f2b13bab5987425558baa5cbdb2bc925b2998038fa028245ef421e75"},
+    {file = "aiohttp-3.10.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8ba01ebc6175e1e6b7275c907a3a36be48a2d487549b656aa90c8a910d9f3178"},
+    {file = "aiohttp-3.10.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8eaf44ccbc4e35762683078b72bf293f476561d8b68ec8a64f98cf32811c323e"},
+    {file = "aiohttp-3.10.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1c43eb1ab7cbf411b8e387dc169acb31f0ca0d8c09ba63f9eac67829585b44f"},
+    {file = "aiohttp-3.10.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:de7a5299827253023c55ea549444e058c0eb496931fa05d693b95140a947cb73"},
+    {file = "aiohttp-3.10.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4790f0e15f00058f7599dab2b206d3049d7ac464dc2e5eae0e93fa18aee9e7bf"},
+    {file = "aiohttp-3.10.5-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:44b324a6b8376a23e6ba25d368726ee3bc281e6ab306db80b5819999c737d820"},
+    {file = "aiohttp-3.10.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:0d277cfb304118079e7044aad0b76685d30ecb86f83a0711fc5fb257ffe832ca"},
+    {file = "aiohttp-3.10.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:54d9ddea424cd19d3ff6128601a4a4d23d54a421f9b4c0fff740505813739a91"},
+    {file = "aiohttp-3.10.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:4f1c9866ccf48a6df2b06823e6ae80573529f2af3a0992ec4fe75b1a510df8a6"},
+    {file = "aiohttp-3.10.5-cp312-cp312-win32.whl", hash = "sha256:dc4826823121783dccc0871e3f405417ac116055bf184ac04c36f98b75aacd12"},
+    {file = "aiohttp-3.10.5-cp312-cp312-win_amd64.whl", hash = "sha256:22c0a23a3b3138a6bf76fc553789cb1a703836da86b0f306b6f0dc1617398abc"},
+    {file = "aiohttp-3.10.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:7f6b639c36734eaa80a6c152a238242bedcee9b953f23bb887e9102976343092"},
+    {file = "aiohttp-3.10.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f29930bc2921cef955ba39a3ff87d2c4398a0394ae217f41cb02d5c26c8b1b77"},
+    {file = "aiohttp-3.10.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f489a2c9e6455d87eabf907ac0b7d230a9786be43fbe884ad184ddf9e9c1e385"},
+    {file = "aiohttp-3.10.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:123dd5b16b75b2962d0fff566effb7a065e33cd4538c1692fb31c3bda2bfb972"},
+    {file = "aiohttp-3.10.5-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b98e698dc34966e5976e10bbca6d26d6724e6bdea853c7c10162a3235aba6e16"},
+    {file = "aiohttp-3.10.5-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c3b9162bab7e42f21243effc822652dc5bb5e8ff42a4eb62fe7782bcbcdfacf6"},
+    {file = "aiohttp-3.10.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1923a5c44061bffd5eebeef58cecf68096e35003907d8201a4d0d6f6e387ccaa"},
+    {file = "aiohttp-3.10.5-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d55f011da0a843c3d3df2c2cf4e537b8070a419f891c930245f05d329c4b0689"},
+    {file = "aiohttp-3.10.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:afe16a84498441d05e9189a15900640a2d2b5e76cf4efe8cbb088ab4f112ee57"},
+    {file = "aiohttp-3.10.5-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:f8112fb501b1e0567a1251a2fd0747baae60a4ab325a871e975b7bb67e59221f"},
+    {file = "aiohttp-3.10.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:1e72589da4c90337837fdfe2026ae1952c0f4a6e793adbbfbdd40efed7c63599"},
+    {file = "aiohttp-3.10.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:4d46c7b4173415d8e583045fbc4daa48b40e31b19ce595b8d92cf639396c15d5"},
+    {file = "aiohttp-3.10.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:33e6bc4bab477c772a541f76cd91e11ccb6d2efa2b8d7d7883591dfb523e5987"},
+    {file = "aiohttp-3.10.5-cp313-cp313-win32.whl", hash = "sha256:c58c6837a2c2a7cf3133983e64173aec11f9c2cd8e87ec2fdc16ce727bcf1a04"},
+    {file = "aiohttp-3.10.5-cp313-cp313-win_amd64.whl", hash = "sha256:38172a70005252b6893088c0f5e8a47d173df7cc2b2bd88650957eb84fcf5022"},
+    {file = "aiohttp-3.10.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:f6f18898ace4bcd2d41a122916475344a87f1dfdec626ecde9ee802a711bc569"},
+    {file = "aiohttp-3.10.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:5ede29d91a40ba22ac1b922ef510aab871652f6c88ef60b9dcdf773c6d32ad7a"},
+    {file = "aiohttp-3.10.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:673f988370f5954df96cc31fd99c7312a3af0a97f09e407399f61583f30da9bc"},
+    {file = "aiohttp-3.10.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:58718e181c56a3c02d25b09d4115eb02aafe1a732ce5714ab70326d9776457c3"},
+    {file = "aiohttp-3.10.5-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4b38b1570242fbab8d86a84128fb5b5234a2f70c2e32f3070143a6d94bc854cf"},
+    {file = "aiohttp-3.10.5-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:074d1bff0163e107e97bd48cad9f928fa5a3eb4b9d33366137ffce08a63e37fe"},
+    {file = "aiohttp-3.10.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd31f176429cecbc1ba499d4aba31aaccfea488f418d60376b911269d3b883c5"},
+    {file = "aiohttp-3.10.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7384d0b87d4635ec38db9263e6a3f1eb609e2e06087f0aa7f63b76833737b471"},
+    {file = "aiohttp-3.10.5-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:8989f46f3d7ef79585e98fa991e6ded55d2f48ae56d2c9fa5e491a6e4effb589"},
+    {file = "aiohttp-3.10.5-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:c83f7a107abb89a227d6c454c613e7606c12a42b9a4ca9c5d7dad25d47c776ae"},
+    {file = "aiohttp-3.10.5-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:cde98f323d6bf161041e7627a5fd763f9fd829bcfcd089804a5fdce7bb6e1b7d"},
+    {file = "aiohttp-3.10.5-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:676f94c5480d8eefd97c0c7e3953315e4d8c2b71f3b49539beb2aa676c58272f"},
+    {file = "aiohttp-3.10.5-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:2d21ac12dc943c68135ff858c3a989f2194a709e6e10b4c8977d7fcd67dfd511"},
+    {file = "aiohttp-3.10.5-cp38-cp38-win32.whl", hash = "sha256:17e997105bd1a260850272bfb50e2a328e029c941c2708170d9d978d5a30ad9a"},
+    {file = "aiohttp-3.10.5-cp38-cp38-win_amd64.whl", hash = "sha256:1c19de68896747a2aa6257ae4cf6ef59d73917a36a35ee9d0a6f48cff0f94db8"},
+    {file = "aiohttp-3.10.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7e2fe37ac654032db1f3499fe56e77190282534810e2a8e833141a021faaab0e"},
+    {file = "aiohttp-3.10.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f5bf3ead3cb66ab990ee2561373b009db5bc0e857549b6c9ba84b20bc462e172"},
+    {file = "aiohttp-3.10.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1b2c16a919d936ca87a3c5f0e43af12a89a3ce7ccbce59a2d6784caba945b68b"},
+    {file = "aiohttp-3.10.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad146dae5977c4dd435eb31373b3fe9b0b1bf26858c6fc452bf6af394067e10b"},
+    {file = "aiohttp-3.10.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8c5c6fa16412b35999320f5c9690c0f554392dc222c04e559217e0f9ae244b92"},
+    {file = "aiohttp-3.10.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:95c4dc6f61d610bc0ee1edc6f29d993f10febfe5b76bb470b486d90bbece6b22"},
+    {file = "aiohttp-3.10.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:da452c2c322e9ce0cfef392e469a26d63d42860f829026a63374fde6b5c5876f"},
+    {file = "aiohttp-3.10.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:898715cf566ec2869d5cb4d5fb4be408964704c46c96b4be267442d265390f32"},
+    {file = "aiohttp-3.10.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:391cc3a9c1527e424c6865e087897e766a917f15dddb360174a70467572ac6ce"},
+    {file = "aiohttp-3.10.5-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:380f926b51b92d02a34119d072f178d80bbda334d1a7e10fa22d467a66e494db"},
+    {file = "aiohttp-3.10.5-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:ce91db90dbf37bb6fa0997f26574107e1b9d5ff939315247b7e615baa8ec313b"},
+    {file = "aiohttp-3.10.5-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:9093a81e18c45227eebe4c16124ebf3e0d893830c6aca7cc310bfca8fe59d857"},
+    {file = "aiohttp-3.10.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:ee40b40aa753d844162dcc80d0fe256b87cba48ca0054f64e68000453caead11"},
+    {file = "aiohttp-3.10.5-cp39-cp39-win32.whl", hash = "sha256:03f2645adbe17f274444953bdea69f8327e9d278d961d85657cb0d06864814c1"},
+    {file = "aiohttp-3.10.5-cp39-cp39-win_amd64.whl", hash = "sha256:d17920f18e6ee090bdd3d0bfffd769d9f2cb4c8ffde3eb203777a3895c128862"},
+    {file = "aiohttp-3.10.5.tar.gz", hash = "sha256:f071854b47d39591ce9a17981c46790acb30518e2f83dfca8db2dfa091178691"},
 ]
 
 [package.dependencies]
+aiohappyeyeballs = ">=2.3.0"
 aiosignal = ">=1.1.2"
-async-timeout = ">=4.0.0a3,<5.0"
+async-timeout = {version = ">=4.0,<5.0", markers = "python_version < \"3.11\""}
 attrs = ">=17.3.0"
-charset-normalizer = ">=2.0,<4.0"
 frozenlist = ">=1.1.1"
 multidict = ">=4.5,<7.0"
 yarl = ">=1.0,<2.0"
 
 [package.extras]
-speedups = ["Brotli", "aiodns", "cchardet"]
+speedups = ["Brotli", "aiodns (>=3.2.0)", "brotlicffi"]
 
 [[package]]
 name = "aiosignal"
@@ -124,13 +139,13 @@ frozenlist = ">=1.1.0"
 
 [[package]]
 name = "annotated-types"
-version = "0.6.0"
+version = "0.7.0"
 description = "Reusable constraint types to use with typing.Annotated"
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "annotated_types-0.6.0-py3-none-any.whl", hash = "sha256:0641064de18ba7a25dee8f96403ebc39113d0cb953a01429249d5c7564666a43"},
-    {file = "annotated_types-0.6.0.tar.gz", hash = "sha256:563339e807e53ffd9c267e99fc6d9ea23eb8443c08f112651963e24e22f84a5d"},
+    {file = "annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53"},
+    {file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"},
 ]
 
 [[package]]
@@ -146,115 +161,131 @@ files = [
 
 [[package]]
 name = "attrs"
-version = "23.1.0"
+version = "24.2.0"
 description = "Classes Without Boilerplate"
 optional = false
 python-versions = ">=3.7"
 files = [
-    {file = "attrs-23.1.0-py3-none-any.whl", hash = "sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04"},
-    {file = "attrs-23.1.0.tar.gz", hash = "sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015"},
+    {file = "attrs-24.2.0-py3-none-any.whl", hash = "sha256:81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2"},
+    {file = "attrs-24.2.0.tar.gz", hash = "sha256:5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346"},
 ]
 
 [package.extras]
-cov = ["attrs[tests]", "coverage[toml] (>=5.3)"]
-dev = ["attrs[docs,tests]", "pre-commit"]
-docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"]
-tests = ["attrs[tests-no-zope]", "zope-interface"]
-tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"]
+benchmark = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-codspeed", "pytest-mypy-plugins", "pytest-xdist[psutil]"]
+cov = ["cloudpickle", "coverage[toml] (>=5.3)", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"]
+dev = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"]
+docs = ["cogapp", "furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier (<24.7)"]
+tests = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"]
+tests-mypy = ["mypy (>=1.11.1)", "pytest-mypy-plugins"]
 
 [[package]]
 name = "certifi"
-version = "2023.7.22"
+version = "2024.8.30"
 description = "Python package for providing Mozilla's CA Bundle."
 optional = false
 python-versions = ">=3.6"
 files = [
-    {file = "certifi-2023.7.22-py3-none-any.whl", hash = "sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"},
-    {file = "certifi-2023.7.22.tar.gz", hash = "sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082"},
+    {file = "certifi-2024.8.30-py3-none-any.whl", hash = "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8"},
+    {file = "certifi-2024.8.30.tar.gz", hash = "sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9"},
 ]
 
 [[package]]
 name = "charset-normalizer"
-version = "3.2.0"
+version = "3.3.2"
 description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet."
 optional = false
 python-versions = ">=3.7.0"
 files = [
-    {file = "charset-normalizer-3.2.0.tar.gz", hash = "sha256:3bb3d25a8e6c0aedd251753a79ae98a093c7e7b471faa3aa9a93a81431987ace"},
-    {file = "charset_normalizer-3.2.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0b87549028f680ca955556e3bd57013ab47474c3124dc069faa0b6545b6c9710"},
-    {file = "charset_normalizer-3.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7c70087bfee18a42b4040bb9ec1ca15a08242cf5867c58726530bdf3945672ed"},
-    {file = "charset_normalizer-3.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a103b3a7069b62f5d4890ae1b8f0597618f628b286b03d4bc9195230b154bfa9"},
-    {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94aea8eff76ee6d1cdacb07dd2123a68283cb5569e0250feab1240058f53b623"},
-    {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:db901e2ac34c931d73054d9797383d0f8009991e723dab15109740a63e7f902a"},
-    {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b0dac0ff919ba34d4df1b6131f59ce95b08b9065233446be7e459f95554c0dc8"},
-    {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:193cbc708ea3aca45e7221ae58f0fd63f933753a9bfb498a3b474878f12caaad"},
-    {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:09393e1b2a9461950b1c9a45d5fd251dc7c6f228acab64da1c9c0165d9c7765c"},
-    {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:baacc6aee0b2ef6f3d308e197b5d7a81c0e70b06beae1f1fcacffdbd124fe0e3"},
-    {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:bf420121d4c8dce6b889f0e8e4ec0ca34b7f40186203f06a946fa0276ba54029"},
-    {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:c04a46716adde8d927adb9457bbe39cf473e1e2c2f5d0a16ceb837e5d841ad4f"},
-    {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:aaf63899c94de41fe3cf934601b0f7ccb6b428c6e4eeb80da72c58eab077b19a"},
-    {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d62e51710986674142526ab9f78663ca2b0726066ae26b78b22e0f5e571238dd"},
-    {file = "charset_normalizer-3.2.0-cp310-cp310-win32.whl", hash = "sha256:04e57ab9fbf9607b77f7d057974694b4f6b142da9ed4a199859d9d4d5c63fe96"},
-    {file = "charset_normalizer-3.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:48021783bdf96e3d6de03a6e39a1171ed5bd7e8bb93fc84cc649d11490f87cea"},
-    {file = "charset_normalizer-3.2.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:4957669ef390f0e6719db3613ab3a7631e68424604a7b448f079bee145da6e09"},
-    {file = "charset_normalizer-3.2.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:46fb8c61d794b78ec7134a715a3e564aafc8f6b5e338417cb19fe9f57a5a9bf2"},
-    {file = "charset_normalizer-3.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f779d3ad205f108d14e99bb3859aa7dd8e9c68874617c72354d7ecaec2a054ac"},
-    {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f25c229a6ba38a35ae6e25ca1264621cc25d4d38dca2942a7fce0b67a4efe918"},
-    {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2efb1bd13885392adfda4614c33d3b68dee4921fd0ac1d3988f8cbb7d589e72a"},
-    {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f30b48dd7fa1474554b0b0f3fdfdd4c13b5c737a3c6284d3cdc424ec0ffff3a"},
-    {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:246de67b99b6851627d945db38147d1b209a899311b1305dd84916f2b88526c6"},
-    {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9bd9b3b31adcb054116447ea22caa61a285d92e94d710aa5ec97992ff5eb7cf3"},
-    {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:8c2f5e83493748286002f9369f3e6607c565a6a90425a3a1fef5ae32a36d749d"},
-    {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:3170c9399da12c9dc66366e9d14da8bf7147e1e9d9ea566067bbce7bb74bd9c2"},
-    {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:7a4826ad2bd6b07ca615c74ab91f32f6c96d08f6fcc3902ceeedaec8cdc3bcd6"},
-    {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:3b1613dd5aee995ec6d4c69f00378bbd07614702a315a2cf6c1d21461fe17c23"},
-    {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9e608aafdb55eb9f255034709e20d5a83b6d60c054df0802fa9c9883d0a937aa"},
-    {file = "charset_normalizer-3.2.0-cp311-cp311-win32.whl", hash = "sha256:f2a1d0fd4242bd8643ce6f98927cf9c04540af6efa92323e9d3124f57727bfc1"},
-    {file = "charset_normalizer-3.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:681eb3d7e02e3c3655d1b16059fbfb605ac464c834a0c629048a30fad2b27489"},
-    {file = "charset_normalizer-3.2.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c57921cda3a80d0f2b8aec7e25c8aa14479ea92b5b51b6876d975d925a2ea346"},
-    {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:41b25eaa7d15909cf3ac4c96088c1f266a9a93ec44f87f1d13d4a0e86c81b982"},
-    {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f058f6963fd82eb143c692cecdc89e075fa0828db2e5b291070485390b2f1c9c"},
-    {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a7647ebdfb9682b7bb97e2a5e7cb6ae735b1c25008a70b906aecca294ee96cf4"},
-    {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eef9df1eefada2c09a5e7a40991b9fc6ac6ef20b1372abd48d2794a316dc0449"},
-    {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e03b8895a6990c9ab2cdcd0f2fe44088ca1c65ae592b8f795c3294af00a461c3"},
-    {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:ee4006268ed33370957f55bf2e6f4d263eaf4dc3cfc473d1d90baff6ed36ce4a"},
-    {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c4983bf937209c57240cff65906b18bb35e64ae872da6a0db937d7b4af845dd7"},
-    {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:3bb7fda7260735efe66d5107fb7e6af6a7c04c7fce9b2514e04b7a74b06bf5dd"},
-    {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:72814c01533f51d68702802d74f77ea026b5ec52793c791e2da806a3844a46c3"},
-    {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:70c610f6cbe4b9fce272c407dd9d07e33e6bf7b4aa1b7ffb6f6ded8e634e3592"},
-    {file = "charset_normalizer-3.2.0-cp37-cp37m-win32.whl", hash = "sha256:a401b4598e5d3f4a9a811f3daf42ee2291790c7f9d74b18d75d6e21dda98a1a1"},
-    {file = "charset_normalizer-3.2.0-cp37-cp37m-win_amd64.whl", hash = "sha256:c0b21078a4b56965e2b12f247467b234734491897e99c1d51cee628da9786959"},
-    {file = "charset_normalizer-3.2.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:95eb302ff792e12aba9a8b8f8474ab229a83c103d74a750ec0bd1c1eea32e669"},
-    {file = "charset_normalizer-3.2.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1a100c6d595a7f316f1b6f01d20815d916e75ff98c27a01ae817439ea7726329"},
-    {file = "charset_normalizer-3.2.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:6339d047dab2780cc6220f46306628e04d9750f02f983ddb37439ca47ced7149"},
-    {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e4b749b9cc6ee664a3300bb3a273c1ca8068c46be705b6c31cf5d276f8628a94"},
-    {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a38856a971c602f98472050165cea2cdc97709240373041b69030be15047691f"},
-    {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f87f746ee241d30d6ed93969de31e5ffd09a2961a051e60ae6bddde9ec3583aa"},
-    {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89f1b185a01fe560bc8ae5f619e924407efca2191b56ce749ec84982fc59a32a"},
-    {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e1c8a2f4c69e08e89632defbfabec2feb8a8d99edc9f89ce33c4b9e36ab63037"},
-    {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2f4ac36d8e2b4cc1aa71df3dd84ff8efbe3bfb97ac41242fbcfc053c67434f46"},
-    {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a386ebe437176aab38c041de1260cd3ea459c6ce5263594399880bbc398225b2"},
-    {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:ccd16eb18a849fd8dcb23e23380e2f0a354e8daa0c984b8a732d9cfaba3a776d"},
-    {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:e6a5bf2cba5ae1bb80b154ed68a3cfa2fa00fde979a7f50d6598d3e17d9ac20c"},
-    {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:45de3f87179c1823e6d9e32156fb14c1927fcc9aba21433f088fdfb555b77c10"},
-    {file = "charset_normalizer-3.2.0-cp38-cp38-win32.whl", hash = "sha256:1000fba1057b92a65daec275aec30586c3de2401ccdcd41f8a5c1e2c87078706"},
-    {file = "charset_normalizer-3.2.0-cp38-cp38-win_amd64.whl", hash = "sha256:8b2c760cfc7042b27ebdb4a43a4453bd829a5742503599144d54a032c5dc7e9e"},
-    {file = "charset_normalizer-3.2.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:855eafa5d5a2034b4621c74925d89c5efef61418570e5ef9b37717d9c796419c"},
-    {file = "charset_normalizer-3.2.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:203f0c8871d5a7987be20c72442488a0b8cfd0f43b7973771640fc593f56321f"},
-    {file = "charset_normalizer-3.2.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e857a2232ba53ae940d3456f7533ce6ca98b81917d47adc3c7fd55dad8fab858"},
-    {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5e86d77b090dbddbe78867a0275cb4df08ea195e660f1f7f13435a4649e954e5"},
-    {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c4fb39a81950ec280984b3a44f5bd12819953dc5fa3a7e6fa7a80db5ee853952"},
-    {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2dee8e57f052ef5353cf608e0b4c871aee320dd1b87d351c28764fc0ca55f9f4"},
-    {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8700f06d0ce6f128de3ccdbc1acaea1ee264d2caa9ca05daaf492fde7c2a7200"},
-    {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1920d4ff15ce893210c1f0c0e9d19bfbecb7983c76b33f046c13a8ffbd570252"},
-    {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:c1c76a1743432b4b60ab3358c937a3fe1341c828ae6194108a94c69028247f22"},
-    {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:f7560358a6811e52e9c4d142d497f1a6e10103d3a6881f18d04dbce3729c0e2c"},
-    {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:c8063cf17b19661471ecbdb3df1c84f24ad2e389e326ccaf89e3fb2484d8dd7e"},
-    {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:cd6dbe0238f7743d0efe563ab46294f54f9bc8f4b9bcf57c3c666cc5bc9d1299"},
-    {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:1249cbbf3d3b04902ff081ffbb33ce3377fa6e4c7356f759f3cd076cc138d020"},
-    {file = "charset_normalizer-3.2.0-cp39-cp39-win32.whl", hash = "sha256:6c409c0deba34f147f77efaa67b8e4bb83d2f11c8806405f76397ae5b8c0d1c9"},
-    {file = "charset_normalizer-3.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:7095f6fbfaa55defb6b733cfeb14efaae7a29f0b59d8cf213be4e7ca0b857b80"},
-    {file = "charset_normalizer-3.2.0-py3-none-any.whl", hash = "sha256:8e098148dd37b4ce3baca71fb394c81dc5d9c7728c95df695d2dca218edf40e6"},
+    {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"},
+    {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"},
+    {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"},
+    {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"},
+    {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"},
+    {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"},
+    {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"},
+    {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"},
+    {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"},
+    {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"},
+    {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"},
+    {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"},
+    {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"},
+    {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"},
+    {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"},
+    {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"},
+    {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"},
+    {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"},
+    {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"},
+    {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"},
+    {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"},
+    {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"},
+    {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"},
+    {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"},
+    {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"},
+    {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"},
+    {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"},
+    {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"},
+    {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"},
+    {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"},
+    {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"},
+    {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"},
+    {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"},
+    {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"},
+    {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"},
+    {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"},
+    {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"},
+    {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"},
+    {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"},
+    {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"},
+    {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"},
+    {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"},
+    {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"},
+    {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"},
+    {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"},
+    {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"},
+    {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"},
+    {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"},
+    {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"},
+    {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"},
+    {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"},
+    {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"},
+    {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"},
+    {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"},
+    {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"},
+    {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"},
+    {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"},
+    {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"},
+    {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"},
+    {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"},
+    {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"},
+    {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"},
+    {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"},
+    {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"},
+    {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"},
+    {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"},
+    {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"},
+    {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"},
+    {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"},
+    {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"},
+    {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"},
+    {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"},
+    {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"},
+    {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"},
+    {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"},
+    {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"},
+    {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"},
+    {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"},
+    {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"},
+    {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"},
+    {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"},
+    {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"},
+    {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"},
+    {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"},
+    {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"},
+    {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"},
+    {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"},
+    {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"},
+    {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"},
+    {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"},
 ]
 
 [[package]]
@@ -268,46 +299,37 @@ files = [
     {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"},
 ]
 
-[[package]]
-name = "colored"
-version = "1.4.4"
-description = "Simple library for color and formatting to terminal"
-optional = false
-python-versions = "*"
-files = [
-    {file = "colored-1.4.4.tar.gz", hash = "sha256:04ff4d4dd514274fe3b99a21bb52fb96f2688c01e93fba7bef37221e7cb56ce0"},
-]
-
 [[package]]
 name = "docker"
-version = "6.1.3"
+version = "7.1.0"
 description = "A Python library for the Docker Engine API."
 optional = false
-python-versions = ">=3.7"
+python-versions = ">=3.8"
 files = [
-    {file = "docker-6.1.3-py3-none-any.whl", hash = "sha256:aecd2277b8bf8e506e484f6ab7aec39abe0038e29fa4a6d3ba86c3fe01844ed9"},
-    {file = "docker-6.1.3.tar.gz", hash = "sha256:aa6d17830045ba5ef0168d5eaa34d37beeb113948c413affe1d5991fc11f9a20"},
+    {file = "docker-7.1.0-py3-none-any.whl", hash = "sha256:c96b93b7f0a746f9e77d325bcfb87422a3d8bd4f03136ae8a85b37f1898d5fc0"},
+    {file = "docker-7.1.0.tar.gz", hash = "sha256:ad8c70e6e3f8926cb8a92619b832b4ea5299e2831c14284663184e200546fa6c"},
 ]
 
 [package.dependencies]
-packaging = ">=14.0"
 pywin32 = {version = ">=304", markers = "sys_platform == \"win32\""}
 requests = ">=2.26.0"
 urllib3 = ">=1.26.0"
-websocket-client = ">=0.32.0"
 
 [package.extras]
+dev = ["coverage (==7.2.7)", "pytest (==7.4.2)", "pytest-cov (==4.1.0)", "pytest-timeout (==2.1.0)", "ruff (==0.1.8)"]
+docs = ["myst-parser (==0.18.0)", "sphinx (==5.1.1)"]
 ssh = ["paramiko (>=2.4.3)"]
+websockets = ["websocket-client (>=1.3.0)"]
 
 [[package]]
 name = "exceptiongroup"
-version = "1.1.3"
+version = "1.2.2"
 description = "Backport of PEP 654 (exception groups)"
 optional = false
 python-versions = ">=3.7"
 files = [
-    {file = "exceptiongroup-1.1.3-py3-none-any.whl", hash = "sha256:343280667a4585d195ca1cf9cef84a4e178c4b6cf2274caef9859782b567d5e3"},
-    {file = "exceptiongroup-1.1.3.tar.gz", hash = "sha256:097acd85d473d75af5bb98e41b61ff7fe35efe6675e4f9370ec6ec5126d160e9"},
+    {file = "exceptiongroup-1.2.2-py3-none-any.whl", hash = "sha256:3111b9d131c238bec2f8f516e123e14ba243563fb135d3fe885990585aa7795b"},
+    {file = "exceptiongroup-1.2.2.tar.gz", hash = "sha256:47c2edf7c6738fafb49fd34290706d1a1a2f4d1c6df275526b62cbb4aa5393cc"},
 ]
 
 [package.extras]
@@ -315,101 +337,115 @@ test = ["pytest (>=6)"]
 
 [[package]]
 name = "filelock"
-version = "3.12.3"
+version = "3.16.0"
 description = "A platform independent file lock."
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "filelock-3.12.3-py3-none-any.whl", hash = "sha256:f067e40ccc40f2b48395a80fcbd4728262fab54e232e090a4063ab804179efeb"},
-    {file = "filelock-3.12.3.tar.gz", hash = "sha256:0ecc1dd2ec4672a10c8550a8182f1bd0c0a5088470ecd5a125e45f49472fac3d"},
+    {file = "filelock-3.16.0-py3-none-any.whl", hash = "sha256:f6ed4c963184f4c84dd5557ce8fece759a3724b37b80c6c4f20a2f63a4dc6609"},
+    {file = "filelock-3.16.0.tar.gz", hash = "sha256:81de9eb8453c769b63369f87f11131a7ab04e367f8d97ad39dc230daa07e3bec"},
 ]
 
-[package.dependencies]
-typing-extensions = {version = ">=4.7.1", markers = "python_version < \"3.11\""}
-
 [package.extras]
-docs = ["furo (>=2023.7.26)", "sphinx (>=7.1.2)", "sphinx-autodoc-typehints (>=1.24)"]
-testing = ["covdefaults (>=2.3)", "coverage (>=7.3)", "diff-cover (>=7.7)", "pytest (>=7.4)", "pytest-cov (>=4.1)", "pytest-mock (>=3.11.1)", "pytest-timeout (>=2.1)"]
+docs = ["furo (>=2024.8.6)", "sphinx (>=8.0.2)", "sphinx-autodoc-typehints (>=2.4)"]
+testing = ["covdefaults (>=2.3)", "coverage (>=7.6.1)", "diff-cover (>=9.1.1)", "pytest (>=8.3.2)", "pytest-asyncio (>=0.24)", "pytest-cov (>=5)", "pytest-mock (>=3.14)", "pytest-timeout (>=2.3.1)", "virtualenv (>=20.26.3)"]
+typing = ["typing-extensions (>=4.12.2)"]
 
 [[package]]
 name = "frozenlist"
-version = "1.4.0"
+version = "1.4.1"
 description = "A list-like structure which implements collections.abc.MutableSequence"
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "frozenlist-1.4.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:764226ceef3125e53ea2cb275000e309c0aa5464d43bd72abd661e27fffc26ab"},
-    {file = "frozenlist-1.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d6484756b12f40003c6128bfcc3fa9f0d49a687e171186c2d85ec82e3758c559"},
-    {file = "frozenlist-1.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9ac08e601308e41eb533f232dbf6b7e4cea762f9f84f6357136eed926c15d12c"},
-    {file = "frozenlist-1.4.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d081f13b095d74b67d550de04df1c756831f3b83dc9881c38985834387487f1b"},
-    {file = "frozenlist-1.4.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:71932b597f9895f011f47f17d6428252fc728ba2ae6024e13c3398a087c2cdea"},
-    {file = "frozenlist-1.4.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:981b9ab5a0a3178ff413bca62526bb784249421c24ad7381e39d67981be2c326"},
-    {file = "frozenlist-1.4.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e41f3de4df3e80de75845d3e743b3f1c4c8613c3997a912dbf0229fc61a8b963"},
-    {file = "frozenlist-1.4.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6918d49b1f90821e93069682c06ffde41829c346c66b721e65a5c62b4bab0300"},
-    {file = "frozenlist-1.4.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:0e5c8764c7829343d919cc2dfc587a8db01c4f70a4ebbc49abde5d4b158b007b"},
-    {file = "frozenlist-1.4.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:8d0edd6b1c7fb94922bf569c9b092ee187a83f03fb1a63076e7774b60f9481a8"},
-    {file = "frozenlist-1.4.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:e29cda763f752553fa14c68fb2195150bfab22b352572cb36c43c47bedba70eb"},
-    {file = "frozenlist-1.4.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:0c7c1b47859ee2cac3846fde1c1dc0f15da6cec5a0e5c72d101e0f83dcb67ff9"},
-    {file = "frozenlist-1.4.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:901289d524fdd571be1c7be054f48b1f88ce8dddcbdf1ec698b27d4b8b9e5d62"},
-    {file = "frozenlist-1.4.0-cp310-cp310-win32.whl", hash = "sha256:1a0848b52815006ea6596c395f87449f693dc419061cc21e970f139d466dc0a0"},
-    {file = "frozenlist-1.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:b206646d176a007466358aa21d85cd8600a415c67c9bd15403336c331a10d956"},
-    {file = "frozenlist-1.4.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:de343e75f40e972bae1ef6090267f8260c1446a1695e77096db6cfa25e759a95"},
-    {file = "frozenlist-1.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ad2a9eb6d9839ae241701d0918f54c51365a51407fd80f6b8289e2dfca977cc3"},
-    {file = "frozenlist-1.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:bd7bd3b3830247580de99c99ea2a01416dfc3c34471ca1298bccabf86d0ff4dc"},
-    {file = "frozenlist-1.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bdf1847068c362f16b353163391210269e4f0569a3c166bc6a9f74ccbfc7e839"},
-    {file = "frozenlist-1.4.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:38461d02d66de17455072c9ba981d35f1d2a73024bee7790ac2f9e361ef1cd0c"},
-    {file = "frozenlist-1.4.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d5a32087d720c608f42caed0ef36d2b3ea61a9d09ee59a5142d6070da9041b8f"},
-    {file = "frozenlist-1.4.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dd65632acaf0d47608190a71bfe46b209719bf2beb59507db08ccdbe712f969b"},
-    {file = "frozenlist-1.4.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:261b9f5d17cac914531331ff1b1d452125bf5daa05faf73b71d935485b0c510b"},
-    {file = "frozenlist-1.4.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b89ac9768b82205936771f8d2eb3ce88503b1556324c9f903e7156669f521472"},
-    {file = "frozenlist-1.4.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:008eb8b31b3ea6896da16c38c1b136cb9fec9e249e77f6211d479db79a4eaf01"},
-    {file = "frozenlist-1.4.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:e74b0506fa5aa5598ac6a975a12aa8928cbb58e1f5ac8360792ef15de1aa848f"},
-    {file = "frozenlist-1.4.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:490132667476f6781b4c9458298b0c1cddf237488abd228b0b3650e5ecba7467"},
-    {file = "frozenlist-1.4.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:76d4711f6f6d08551a7e9ef28c722f4a50dd0fc204c56b4bcd95c6cc05ce6fbb"},
-    {file = "frozenlist-1.4.0-cp311-cp311-win32.whl", hash = "sha256:a02eb8ab2b8f200179b5f62b59757685ae9987996ae549ccf30f983f40602431"},
-    {file = "frozenlist-1.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:515e1abc578dd3b275d6a5114030b1330ba044ffba03f94091842852f806f1c1"},
-    {file = "frozenlist-1.4.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:f0ed05f5079c708fe74bf9027e95125334b6978bf07fd5ab923e9e55e5fbb9d3"},
-    {file = "frozenlist-1.4.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ca265542ca427bf97aed183c1676e2a9c66942e822b14dc6e5f42e038f92a503"},
-    {file = "frozenlist-1.4.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:491e014f5c43656da08958808588cc6c016847b4360e327a62cb308c791bd2d9"},
-    {file = "frozenlist-1.4.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:17ae5cd0f333f94f2e03aaf140bb762c64783935cc764ff9c82dff626089bebf"},
-    {file = "frozenlist-1.4.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1e78fb68cf9c1a6aa4a9a12e960a5c9dfbdb89b3695197aa7064705662515de2"},
-    {file = "frozenlist-1.4.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d5655a942f5f5d2c9ed93d72148226d75369b4f6952680211972a33e59b1dfdc"},
-    {file = "frozenlist-1.4.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c11b0746f5d946fecf750428a95f3e9ebe792c1ee3b1e96eeba145dc631a9672"},
-    {file = "frozenlist-1.4.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e66d2a64d44d50d2543405fb183a21f76b3b5fd16f130f5c99187c3fb4e64919"},
-    {file = "frozenlist-1.4.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:88f7bc0fcca81f985f78dd0fa68d2c75abf8272b1f5c323ea4a01a4d7a614efc"},
-    {file = "frozenlist-1.4.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:5833593c25ac59ede40ed4de6d67eb42928cca97f26feea219f21d0ed0959b79"},
-    {file = "frozenlist-1.4.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:fec520865f42e5c7f050c2a79038897b1c7d1595e907a9e08e3353293ffc948e"},
-    {file = "frozenlist-1.4.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:b826d97e4276750beca7c8f0f1a4938892697a6bcd8ec8217b3312dad6982781"},
-    {file = "frozenlist-1.4.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:ceb6ec0a10c65540421e20ebd29083c50e6d1143278746a4ef6bcf6153171eb8"},
-    {file = "frozenlist-1.4.0-cp38-cp38-win32.whl", hash = "sha256:2b8bcf994563466db019fab287ff390fffbfdb4f905fc77bc1c1d604b1c689cc"},
-    {file = "frozenlist-1.4.0-cp38-cp38-win_amd64.whl", hash = "sha256:a6c8097e01886188e5be3e6b14e94ab365f384736aa1fca6a0b9e35bd4a30bc7"},
-    {file = "frozenlist-1.4.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:6c38721585f285203e4b4132a352eb3daa19121a035f3182e08e437cface44bf"},
-    {file = "frozenlist-1.4.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a0c6da9aee33ff0b1a451e867da0c1f47408112b3391dd43133838339e410963"},
-    {file = "frozenlist-1.4.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:93ea75c050c5bb3d98016b4ba2497851eadf0ac154d88a67d7a6816206f6fa7f"},
-    {file = "frozenlist-1.4.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f61e2dc5ad442c52b4887f1fdc112f97caeff4d9e6ebe78879364ac59f1663e1"},
-    {file = "frozenlist-1.4.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aa384489fefeb62321b238e64c07ef48398fe80f9e1e6afeff22e140e0850eef"},
-    {file = "frozenlist-1.4.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:10ff5faaa22786315ef57097a279b833ecab1a0bfb07d604c9cbb1c4cdc2ed87"},
-    {file = "frozenlist-1.4.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:007df07a6e3eb3e33e9a1fe6a9db7af152bbd8a185f9aaa6ece10a3529e3e1c6"},
-    {file = "frozenlist-1.4.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f4f399d28478d1f604c2ff9119907af9726aed73680e5ed1ca634d377abb087"},
-    {file = "frozenlist-1.4.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:c5374b80521d3d3f2ec5572e05adc94601985cc526fb276d0c8574a6d749f1b3"},
-    {file = "frozenlist-1.4.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:ce31ae3e19f3c902de379cf1323d90c649425b86de7bbdf82871b8a2a0615f3d"},
-    {file = "frozenlist-1.4.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7211ef110a9194b6042449431e08c4d80c0481e5891e58d429df5899690511c2"},
-    {file = "frozenlist-1.4.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:556de4430ce324c836789fa4560ca62d1591d2538b8ceb0b4f68fb7b2384a27a"},
-    {file = "frozenlist-1.4.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7645a8e814a3ee34a89c4a372011dcd817964ce8cb273c8ed6119d706e9613e3"},
-    {file = "frozenlist-1.4.0-cp39-cp39-win32.whl", hash = "sha256:19488c57c12d4e8095a922f328df3f179c820c212940a498623ed39160bc3c2f"},
-    {file = "frozenlist-1.4.0-cp39-cp39-win_amd64.whl", hash = "sha256:6221d84d463fb110bdd7619b69cb43878a11d51cbb9394ae3105d082d5199167"},
-    {file = "frozenlist-1.4.0.tar.gz", hash = "sha256:09163bdf0b2907454042edb19f887c6d33806adc71fbd54afc14908bfdc22251"},
+    {file = "frozenlist-1.4.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:f9aa1878d1083b276b0196f2dfbe00c9b7e752475ed3b682025ff20c1c1f51ac"},
+    {file = "frozenlist-1.4.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:29acab3f66f0f24674b7dc4736477bcd4bc3ad4b896f5f45379a67bce8b96868"},
+    {file = "frozenlist-1.4.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:74fb4bee6880b529a0c6560885fce4dc95936920f9f20f53d99a213f7bf66776"},
+    {file = "frozenlist-1.4.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:590344787a90ae57d62511dd7c736ed56b428f04cd8c161fcc5e7232c130c69a"},
+    {file = "frozenlist-1.4.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:068b63f23b17df8569b7fdca5517edef76171cf3897eb68beb01341131fbd2ad"},
+    {file = "frozenlist-1.4.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5c849d495bf5154cd8da18a9eb15db127d4dba2968d88831aff6f0331ea9bd4c"},
+    {file = "frozenlist-1.4.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9750cc7fe1ae3b1611bb8cfc3f9ec11d532244235d75901fb6b8e42ce9229dfe"},
+    {file = "frozenlist-1.4.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a9b2de4cf0cdd5bd2dee4c4f63a653c61d2408055ab77b151c1957f221cabf2a"},
+    {file = "frozenlist-1.4.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:0633c8d5337cb5c77acbccc6357ac49a1770b8c487e5b3505c57b949b4b82e98"},
+    {file = "frozenlist-1.4.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:27657df69e8801be6c3638054e202a135c7f299267f1a55ed3a598934f6c0d75"},
+    {file = "frozenlist-1.4.1-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:f9a3ea26252bd92f570600098783d1371354d89d5f6b7dfd87359d669f2109b5"},
+    {file = "frozenlist-1.4.1-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:4f57dab5fe3407b6c0c1cc907ac98e8a189f9e418f3b6e54d65a718aaafe3950"},
+    {file = "frozenlist-1.4.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:e02a0e11cf6597299b9f3bbd3f93d79217cb90cfd1411aec33848b13f5c656cc"},
+    {file = "frozenlist-1.4.1-cp310-cp310-win32.whl", hash = "sha256:a828c57f00f729620a442881cc60e57cfcec6842ba38e1b19fd3e47ac0ff8dc1"},
+    {file = "frozenlist-1.4.1-cp310-cp310-win_amd64.whl", hash = "sha256:f56e2333dda1fe0f909e7cc59f021eba0d2307bc6f012a1ccf2beca6ba362439"},
+    {file = "frozenlist-1.4.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:a0cb6f11204443f27a1628b0e460f37fb30f624be6051d490fa7d7e26d4af3d0"},
+    {file = "frozenlist-1.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b46c8ae3a8f1f41a0d2ef350c0b6e65822d80772fe46b653ab6b6274f61d4a49"},
+    {file = "frozenlist-1.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fde5bd59ab5357e3853313127f4d3565fc7dad314a74d7b5d43c22c6a5ed2ced"},
+    {file = "frozenlist-1.4.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:722e1124aec435320ae01ee3ac7bec11a5d47f25d0ed6328f2273d287bc3abb0"},
+    {file = "frozenlist-1.4.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2471c201b70d58a0f0c1f91261542a03d9a5e088ed3dc6c160d614c01649c106"},
+    {file = "frozenlist-1.4.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c757a9dd70d72b076d6f68efdbb9bc943665ae954dad2801b874c8c69e185068"},
+    {file = "frozenlist-1.4.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f146e0911cb2f1da549fc58fc7bcd2b836a44b79ef871980d605ec392ff6b0d2"},
+    {file = "frozenlist-1.4.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f9c515e7914626b2a2e1e311794b4c35720a0be87af52b79ff8e1429fc25f19"},
+    {file = "frozenlist-1.4.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:c302220494f5c1ebeb0912ea782bcd5e2f8308037b3c7553fad0e48ebad6ad82"},
+    {file = "frozenlist-1.4.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:442acde1e068288a4ba7acfe05f5f343e19fac87bfc96d89eb886b0363e977ec"},
+    {file = "frozenlist-1.4.1-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:1b280e6507ea8a4fa0c0a7150b4e526a8d113989e28eaaef946cc77ffd7efc0a"},
+    {file = "frozenlist-1.4.1-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:fe1a06da377e3a1062ae5fe0926e12b84eceb8a50b350ddca72dc85015873f74"},
+    {file = "frozenlist-1.4.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:db9e724bebd621d9beca794f2a4ff1d26eed5965b004a97f1f1685a173b869c2"},
+    {file = "frozenlist-1.4.1-cp311-cp311-win32.whl", hash = "sha256:e774d53b1a477a67838a904131c4b0eef6b3d8a651f8b138b04f748fccfefe17"},
+    {file = "frozenlist-1.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:fb3c2db03683b5767dedb5769b8a40ebb47d6f7f45b1b3e3b4b51ec8ad9d9825"},
+    {file = "frozenlist-1.4.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:1979bc0aeb89b33b588c51c54ab0161791149f2461ea7c7c946d95d5f93b56ae"},
+    {file = "frozenlist-1.4.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:cc7b01b3754ea68a62bd77ce6020afaffb44a590c2289089289363472d13aedb"},
+    {file = "frozenlist-1.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c9c92be9fd329ac801cc420e08452b70e7aeab94ea4233a4804f0915c14eba9b"},
+    {file = "frozenlist-1.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5c3894db91f5a489fc8fa6a9991820f368f0b3cbdb9cd8849547ccfab3392d86"},
+    {file = "frozenlist-1.4.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ba60bb19387e13597fb059f32cd4d59445d7b18b69a745b8f8e5db0346f33480"},
+    {file = "frozenlist-1.4.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8aefbba5f69d42246543407ed2461db31006b0f76c4e32dfd6f42215a2c41d09"},
+    {file = "frozenlist-1.4.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:780d3a35680ced9ce682fbcf4cb9c2bad3136eeff760ab33707b71db84664e3a"},
+    {file = "frozenlist-1.4.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9acbb16f06fe7f52f441bb6f413ebae6c37baa6ef9edd49cdd567216da8600cd"},
+    {file = "frozenlist-1.4.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:23b701e65c7b36e4bf15546a89279bd4d8675faabc287d06bbcfac7d3c33e1e6"},
+    {file = "frozenlist-1.4.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:3e0153a805a98f5ada7e09826255ba99fb4f7524bb81bf6b47fb702666484ae1"},
+    {file = "frozenlist-1.4.1-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:dd9b1baec094d91bf36ec729445f7769d0d0cf6b64d04d86e45baf89e2b9059b"},
+    {file = "frozenlist-1.4.1-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:1a4471094e146b6790f61b98616ab8e44f72661879cc63fa1049d13ef711e71e"},
+    {file = "frozenlist-1.4.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:5667ed53d68d91920defdf4035d1cdaa3c3121dc0b113255124bcfada1cfa1b8"},
+    {file = "frozenlist-1.4.1-cp312-cp312-win32.whl", hash = "sha256:beee944ae828747fd7cb216a70f120767fc9f4f00bacae8543c14a6831673f89"},
+    {file = "frozenlist-1.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:64536573d0a2cb6e625cf309984e2d873979709f2cf22839bf2d61790b448ad5"},
+    {file = "frozenlist-1.4.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:20b51fa3f588ff2fe658663db52a41a4f7aa6c04f6201449c6c7c476bd255c0d"},
+    {file = "frozenlist-1.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:410478a0c562d1a5bcc2f7ea448359fcb050ed48b3c6f6f4f18c313a9bdb1826"},
+    {file = "frozenlist-1.4.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c6321c9efe29975232da3bd0af0ad216800a47e93d763ce64f291917a381b8eb"},
+    {file = "frozenlist-1.4.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:48f6a4533887e189dae092f1cf981f2e3885175f7a0f33c91fb5b7b682b6bab6"},
+    {file = "frozenlist-1.4.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6eb73fa5426ea69ee0e012fb59cdc76a15b1283d6e32e4f8dc4482ec67d1194d"},
+    {file = "frozenlist-1.4.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fbeb989b5cc29e8daf7f976b421c220f1b8c731cbf22b9130d8815418ea45887"},
+    {file = "frozenlist-1.4.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:32453c1de775c889eb4e22f1197fe3bdfe457d16476ea407472b9442e6295f7a"},
+    {file = "frozenlist-1.4.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:693945278a31f2086d9bf3df0fe8254bbeaef1fe71e1351c3bd730aa7d31c41b"},
+    {file = "frozenlist-1.4.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:1d0ce09d36d53bbbe566fe296965b23b961764c0bcf3ce2fa45f463745c04701"},
+    {file = "frozenlist-1.4.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:3a670dc61eb0d0eb7080890c13de3066790f9049b47b0de04007090807c776b0"},
+    {file = "frozenlist-1.4.1-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:dca69045298ce5c11fd539682cff879cc1e664c245d1c64da929813e54241d11"},
+    {file = "frozenlist-1.4.1-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:a06339f38e9ed3a64e4c4e43aec7f59084033647f908e4259d279a52d3757d09"},
+    {file = "frozenlist-1.4.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b7f2f9f912dca3934c1baec2e4585a674ef16fe00218d833856408c48d5beee7"},
+    {file = "frozenlist-1.4.1-cp38-cp38-win32.whl", hash = "sha256:e7004be74cbb7d9f34553a5ce5fb08be14fb33bc86f332fb71cbe5216362a497"},
+    {file = "frozenlist-1.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:5a7d70357e7cee13f470c7883a063aae5fe209a493c57d86eb7f5a6f910fae09"},
+    {file = "frozenlist-1.4.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:bfa4a17e17ce9abf47a74ae02f32d014c5e9404b6d9ac7f729e01562bbee601e"},
+    {file = "frozenlist-1.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b7e3ed87d4138356775346e6845cccbe66cd9e207f3cd11d2f0b9fd13681359d"},
+    {file = "frozenlist-1.4.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c99169d4ff810155ca50b4da3b075cbde79752443117d89429595c2e8e37fed8"},
+    {file = "frozenlist-1.4.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:edb678da49d9f72c9f6c609fbe41a5dfb9a9282f9e6a2253d5a91e0fc382d7c0"},
+    {file = "frozenlist-1.4.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6db4667b187a6742b33afbbaf05a7bc551ffcf1ced0000a571aedbb4aa42fc7b"},
+    {file = "frozenlist-1.4.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:55fdc093b5a3cb41d420884cdaf37a1e74c3c37a31f46e66286d9145d2063bd0"},
+    {file = "frozenlist-1.4.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:82e8211d69a4f4bc360ea22cd6555f8e61a1bd211d1d5d39d3d228b48c83a897"},
+    {file = "frozenlist-1.4.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89aa2c2eeb20957be2d950b85974b30a01a762f3308cd02bb15e1ad632e22dc7"},
+    {file = "frozenlist-1.4.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:9d3e0c25a2350080e9319724dede4f31f43a6c9779be48021a7f4ebde8b2d742"},
+    {file = "frozenlist-1.4.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7268252af60904bf52c26173cbadc3a071cece75f873705419c8681f24d3edea"},
+    {file = "frozenlist-1.4.1-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:0c250a29735d4f15321007fb02865f0e6b6a41a6b88f1f523ca1596ab5f50bd5"},
+    {file = "frozenlist-1.4.1-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:96ec70beabbd3b10e8bfe52616a13561e58fe84c0101dd031dc78f250d5128b9"},
+    {file = "frozenlist-1.4.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:23b2d7679b73fe0e5a4560b672a39f98dfc6f60df63823b0a9970525325b95f6"},
+    {file = "frozenlist-1.4.1-cp39-cp39-win32.whl", hash = "sha256:a7496bfe1da7fb1a4e1cc23bb67c58fab69311cc7d32b5a99c2007b4b2a0e932"},
+    {file = "frozenlist-1.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:e6a20a581f9ce92d389a8c7d7c3dd47c81fd5d6e655c8dddf341e14aa48659d0"},
+    {file = "frozenlist-1.4.1-py3-none-any.whl", hash = "sha256:04ced3e6a46b4cfffe20f9ae482818e34eba9b5fb0ce4056e4cc9b6e212d09b7"},
+    {file = "frozenlist-1.4.1.tar.gz", hash = "sha256:c037a86e8513059a2613aaba4d817bb90b9d9b6b69aace3ce9c877e8c8ed402b"},
 ]
 
 [[package]]
 name = "fsspec"
-version = "2023.6.0"
+version = "2024.9.0"
 description = "File-system specification"
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "fsspec-2023.6.0-py3-none-any.whl", hash = "sha256:1cbad1faef3e391fba6dc005ae9b5bdcbf43005c9167ce78c915549c352c869a"},
-    {file = "fsspec-2023.6.0.tar.gz", hash = "sha256:d0b2f935446169753e7a5c5c55681c54ea91996cc67be93c39a154fb3a2742af"},
+    {file = "fsspec-2024.9.0-py3-none-any.whl", hash = "sha256:a0947d552d8a6efa72cc2c730b12c41d043509156966cca4fb157b0f2a0c574b"},
+    {file = "fsspec-2024.9.0.tar.gz", hash = "sha256:4b0afb90c2f21832df142f292649035d80b421f60a9e1c027802e5a0da2b04e8"},
 ]
 
 [package.extras]
@@ -417,7 +453,8 @@ abfs = ["adlfs"]
 adl = ["adlfs"]
 arrow = ["pyarrow (>=1)"]
 dask = ["dask", "distributed"]
-devel = ["pytest", "pytest-cov"]
+dev = ["pre-commit", "ruff"]
+doc = ["numpydoc", "sphinx", "sphinx-design", "sphinx-rtd-theme", "yarl"]
 dropbox = ["dropbox", "dropboxdrivefs", "requests"]
 full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "dask", "distributed", "dropbox", "dropboxdrivefs", "fusepy", "gcsfs", "libarchive-c", "ocifs", "panel", "paramiko", "pyarrow (>=1)", "pygit2", "requests", "s3fs", "smbprotocol", "tqdm"]
 fuse = ["fusepy"]
@@ -427,29 +464,32 @@ github = ["requests"]
 gs = ["gcsfs"]
 gui = ["panel"]
 hdfs = ["pyarrow (>=1)"]
-http = ["aiohttp (!=4.0.0a0,!=4.0.0a1)", "requests"]
+http = ["aiohttp (!=4.0.0a0,!=4.0.0a1)"]
 libarchive = ["libarchive-c"]
 oci = ["ocifs"]
 s3 = ["s3fs"]
 sftp = ["paramiko"]
 smb = ["smbprotocol"]
 ssh = ["paramiko"]
+test = ["aiohttp (!=4.0.0a0,!=4.0.0a1)", "numpy", "pytest", "pytest-asyncio (!=0.22.0)", "pytest-benchmark", "pytest-cov", "pytest-mock", "pytest-recording", "pytest-rerunfailures", "requests"]
+test-downstream = ["aiobotocore (>=2.5.4,<3.0.0)", "dask-expr", "dask[dataframe,test]", "moto[server] (>4,<5)", "pytest-timeout", "xarray"]
+test-full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "cloudpickle", "dask", "distributed", "dropbox", "dropboxdrivefs", "fastparquet", "fusepy", "gcsfs", "jinja2", "kerchunk", "libarchive-c", "lz4", "notebook", "numpy", "ocifs", "pandas", "panel", "paramiko", "pyarrow", "pyarrow (>=1)", "pyftpdlib", "pygit2", "pytest", "pytest-asyncio (!=0.22.0)", "pytest-benchmark", "pytest-cov", "pytest-mock", "pytest-recording", "pytest-rerunfailures", "python-snappy", "requests", "smbprotocol", "tqdm", "urllib3", "zarr", "zstandard"]
 tqdm = ["tqdm"]
 
 [[package]]
 name = "huggingface-hub"
-version = "0.16.4"
+version = "0.24.6"
 description = "Client library to download and publish models, datasets and other repos on the huggingface.co hub"
 optional = false
-python-versions = ">=3.7.0"
+python-versions = ">=3.8.0"
 files = [
-    {file = "huggingface_hub-0.16.4-py3-none-any.whl", hash = "sha256:0d3df29932f334fead024afc7cb4cc5149d955238b8b5e42dcf9740d6995a349"},
-    {file = "huggingface_hub-0.16.4.tar.gz", hash = "sha256:608c7d4f3d368b326d1747f91523dbd1f692871e8e2e7a4750314a2dd8b63e14"},
+    {file = "huggingface_hub-0.24.6-py3-none-any.whl", hash = "sha256:a990f3232aa985fe749bc9474060cbad75e8b2f115f6665a9fda5b9c97818970"},
+    {file = "huggingface_hub-0.24.6.tar.gz", hash = "sha256:cc2579e761d070713eaa9c323e3debe39d5b464ae3a7261c39a9195b27bb8000"},
 ]
 
 [package.dependencies]
 filelock = "*"
-fsspec = "*"
+fsspec = ">=2023.5.0"
 packaging = ">=20.9"
 pyyaml = ">=5.1"
 requests = "*"
@@ -457,26 +497,28 @@ tqdm = ">=4.42.1"
 typing-extensions = ">=3.7.4.3"
 
 [package.extras]
-all = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "black (>=23.1,<24.0)", "gradio", "jedi", "mypy (==0.982)", "numpy", "pydantic", "pytest", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-vcr", "pytest-xdist", "ruff (>=0.0.241)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "urllib3 (<2.0)"]
+all = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "fastapi", "gradio", "jedi", "minijinja (>=1.0)", "mypy (==1.5.1)", "numpy", "pytest (>=8.1.1,<8.2.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures", "pytest-vcr", "pytest-xdist", "ruff (>=0.5.0)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"]
 cli = ["InquirerPy (==0.3.4)"]
-dev = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "black (>=23.1,<24.0)", "gradio", "jedi", "mypy (==0.982)", "numpy", "pydantic", "pytest", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-vcr", "pytest-xdist", "ruff (>=0.0.241)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "urllib3 (<2.0)"]
+dev = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "fastapi", "gradio", "jedi", "minijinja (>=1.0)", "mypy (==1.5.1)", "numpy", "pytest (>=8.1.1,<8.2.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures", "pytest-vcr", "pytest-xdist", "ruff (>=0.5.0)", "soundfile", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)", "urllib3 (<2.0)"]
 fastai = ["fastai (>=2.4)", "fastcore (>=1.3.27)", "toml"]
-inference = ["aiohttp", "pydantic"]
-quality = ["black (>=23.1,<24.0)", "mypy (==0.982)", "ruff (>=0.0.241)"]
+hf-transfer = ["hf-transfer (>=0.1.4)"]
+inference = ["aiohttp", "minijinja (>=1.0)"]
+quality = ["mypy (==1.5.1)", "ruff (>=0.5.0)"]
 tensorflow = ["graphviz", "pydot", "tensorflow"]
-testing = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "gradio", "jedi", "numpy", "pydantic", "pytest", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-vcr", "pytest-xdist", "soundfile", "urllib3 (<2.0)"]
-torch = ["torch"]
-typing = ["pydantic", "types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3"]
+tensorflow-testing = ["keras (<3.0)", "tensorflow"]
+testing = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "fastapi", "gradio", "jedi", "minijinja (>=1.0)", "numpy", "pytest (>=8.1.1,<8.2.2)", "pytest-asyncio", "pytest-cov", "pytest-env", "pytest-mock", "pytest-rerunfailures", "pytest-vcr", "pytest-xdist", "soundfile", "urllib3 (<2.0)"]
+torch = ["safetensors[torch]", "torch"]
+typing = ["types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)"]
 
 [[package]]
 name = "idna"
-version = "3.4"
+version = "3.8"
 description = "Internationalized Domain Names in Applications (IDNA)"
 optional = false
-python-versions = ">=3.5"
+python-versions = ">=3.6"
 files = [
-    {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"},
-    {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"},
+    {file = "idna-3.8-py3-none-any.whl", hash = "sha256:050b4e5baadcd44d760cedbd2b8e639f2ff89bbc7a5730fcc662954303377aac"},
+    {file = "idna-3.8.tar.gz", hash = "sha256:d838c2c0ed6fced7693d5e8ab8e734d5f8fda53a039c0164afb0b82e771e3603"},
 ]
 
 [[package]]
@@ -492,107 +534,173 @@ files = [
 
 [[package]]
 name = "multidict"
-version = "6.0.4"
+version = "6.1.0"
 description = "multidict implementation"
 optional = false
-python-versions = ">=3.7"
+python-versions = ">=3.8"
 files = [
-    {file = "multidict-6.0.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0b1a97283e0c85772d613878028fec909f003993e1007eafa715b24b377cb9b8"},
-    {file = "multidict-6.0.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:eeb6dcc05e911516ae3d1f207d4b0520d07f54484c49dfc294d6e7d63b734171"},
-    {file = "multidict-6.0.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d6d635d5209b82a3492508cf5b365f3446afb65ae7ebd755e70e18f287b0adf7"},
-    {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c048099e4c9e9d615545e2001d3d8a4380bd403e1a0578734e0d31703d1b0c0b"},
-    {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ea20853c6dbbb53ed34cb4d080382169b6f4554d394015f1bef35e881bf83547"},
-    {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:16d232d4e5396c2efbbf4f6d4df89bfa905eb0d4dc5b3549d872ab898451f569"},
-    {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:36c63aaa167f6c6b04ef2c85704e93af16c11d20de1d133e39de6a0e84582a93"},
-    {file = "multidict-6.0.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:64bdf1086b6043bf519869678f5f2757f473dee970d7abf6da91ec00acb9cb98"},
-    {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:43644e38f42e3af682690876cff722d301ac585c5b9e1eacc013b7a3f7b696a0"},
-    {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:7582a1d1030e15422262de9f58711774e02fa80df0d1578995c76214f6954988"},
-    {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:ddff9c4e225a63a5afab9dd15590432c22e8057e1a9a13d28ed128ecf047bbdc"},
-    {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:ee2a1ece51b9b9e7752e742cfb661d2a29e7bcdba2d27e66e28a99f1890e4fa0"},
-    {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a2e4369eb3d47d2034032a26c7a80fcb21a2cb22e1173d761a162f11e562caa5"},
-    {file = "multidict-6.0.4-cp310-cp310-win32.whl", hash = "sha256:574b7eae1ab267e5f8285f0fe881f17efe4b98c39a40858247720935b893bba8"},
-    {file = "multidict-6.0.4-cp310-cp310-win_amd64.whl", hash = "sha256:4dcbb0906e38440fa3e325df2359ac6cb043df8e58c965bb45f4e406ecb162cc"},
-    {file = "multidict-6.0.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0dfad7a5a1e39c53ed00d2dd0c2e36aed4650936dc18fd9a1826a5ae1cad6f03"},
-    {file = "multidict-6.0.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:64da238a09d6039e3bd39bb3aee9c21a5e34f28bfa5aa22518581f910ff94af3"},
-    {file = "multidict-6.0.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ff959bee35038c4624250473988b24f846cbeb2c6639de3602c073f10410ceba"},
-    {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:01a3a55bd90018c9c080fbb0b9f4891db37d148a0a18722b42f94694f8b6d4c9"},
-    {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c5cb09abb18c1ea940fb99360ea0396f34d46566f157122c92dfa069d3e0e982"},
-    {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:666daae833559deb2d609afa4490b85830ab0dfca811a98b70a205621a6109fe"},
-    {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:11bdf3f5e1518b24530b8241529d2050014c884cf18b6fc69c0c2b30ca248710"},
-    {file = "multidict-6.0.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7d18748f2d30f94f498e852c67d61261c643b349b9d2a581131725595c45ec6c"},
-    {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:458f37be2d9e4c95e2d8866a851663cbc76e865b78395090786f6cd9b3bbf4f4"},
-    {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:b1a2eeedcead3a41694130495593a559a668f382eee0727352b9a41e1c45759a"},
-    {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:7d6ae9d593ef8641544d6263c7fa6408cc90370c8cb2bbb65f8d43e5b0351d9c"},
-    {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:5979b5632c3e3534e42ca6ff856bb24b2e3071b37861c2c727ce220d80eee9ed"},
-    {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:dcfe792765fab89c365123c81046ad4103fcabbc4f56d1c1997e6715e8015461"},
-    {file = "multidict-6.0.4-cp311-cp311-win32.whl", hash = "sha256:3601a3cece3819534b11d4efc1eb76047488fddd0c85a3948099d5da4d504636"},
-    {file = "multidict-6.0.4-cp311-cp311-win_amd64.whl", hash = "sha256:81a4f0b34bd92df3da93315c6a59034df95866014ac08535fc819f043bfd51f0"},
-    {file = "multidict-6.0.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:67040058f37a2a51ed8ea8f6b0e6ee5bd78ca67f169ce6122f3e2ec80dfe9b78"},
-    {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:853888594621e6604c978ce2a0444a1e6e70c8d253ab65ba11657659dcc9100f"},
-    {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:39ff62e7d0f26c248b15e364517a72932a611a9b75f35b45be078d81bdb86603"},
-    {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:af048912e045a2dc732847d33821a9d84ba553f5c5f028adbd364dd4765092ac"},
-    {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1e8b901e607795ec06c9e42530788c45ac21ef3aaa11dbd0c69de543bfb79a9"},
-    {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:62501642008a8b9871ddfccbf83e4222cf8ac0d5aeedf73da36153ef2ec222d2"},
-    {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:99b76c052e9f1bc0721f7541e5e8c05db3941eb9ebe7b8553c625ef88d6eefde"},
-    {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:509eac6cf09c794aa27bcacfd4d62c885cce62bef7b2c3e8b2e49d365b5003fe"},
-    {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:21a12c4eb6ddc9952c415f24eef97e3e55ba3af61f67c7bc388dcdec1404a067"},
-    {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:5cad9430ab3e2e4fa4a2ef4450f548768400a2ac635841bc2a56a2052cdbeb87"},
-    {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ab55edc2e84460694295f401215f4a58597f8f7c9466faec545093045476327d"},
-    {file = "multidict-6.0.4-cp37-cp37m-win32.whl", hash = "sha256:5a4dcf02b908c3b8b17a45fb0f15b695bf117a67b76b7ad18b73cf8e92608775"},
-    {file = "multidict-6.0.4-cp37-cp37m-win_amd64.whl", hash = "sha256:6ed5f161328b7df384d71b07317f4d8656434e34591f20552c7bcef27b0ab88e"},
-    {file = "multidict-6.0.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5fc1b16f586f049820c5c5b17bb4ee7583092fa0d1c4e28b5239181ff9532e0c"},
-    {file = "multidict-6.0.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1502e24330eb681bdaa3eb70d6358e818e8e8f908a22a1851dfd4e15bc2f8161"},
-    {file = "multidict-6.0.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b692f419760c0e65d060959df05f2a531945af31fda0c8a3b3195d4efd06de11"},
-    {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45e1ecb0379bfaab5eef059f50115b54571acfbe422a14f668fc8c27ba410e7e"},
-    {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ddd3915998d93fbcd2566ddf9cf62cdb35c9e093075f862935573d265cf8f65d"},
-    {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:59d43b61c59d82f2effb39a93c48b845efe23a3852d201ed2d24ba830d0b4cf2"},
-    {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cc8e1d0c705233c5dd0c5e6460fbad7827d5d36f310a0fadfd45cc3029762258"},
-    {file = "multidict-6.0.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d6aa0418fcc838522256761b3415822626f866758ee0bc6632c9486b179d0b52"},
-    {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6748717bb10339c4760c1e63da040f5f29f5ed6e59d76daee30305894069a660"},
-    {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:4d1a3d7ef5e96b1c9e92f973e43aa5e5b96c659c9bc3124acbbd81b0b9c8a951"},
-    {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4372381634485bec7e46718edc71528024fcdc6f835baefe517b34a33c731d60"},
-    {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:fc35cb4676846ef752816d5be2193a1e8367b4c1397b74a565a9d0389c433a1d"},
-    {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4b9d9e4e2b37daddb5c23ea33a3417901fa7c7b3dee2d855f63ee67a0b21e5b1"},
-    {file = "multidict-6.0.4-cp38-cp38-win32.whl", hash = "sha256:e41b7e2b59679edfa309e8db64fdf22399eec4b0b24694e1b2104fb789207779"},
-    {file = "multidict-6.0.4-cp38-cp38-win_amd64.whl", hash = "sha256:d6c254ba6e45d8e72739281ebc46ea5eb5f101234f3ce171f0e9f5cc86991480"},
-    {file = "multidict-6.0.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:16ab77bbeb596e14212e7bab8429f24c1579234a3a462105cda4a66904998664"},
-    {file = "multidict-6.0.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bc779e9e6f7fda81b3f9aa58e3a6091d49ad528b11ed19f6621408806204ad35"},
-    {file = "multidict-6.0.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4ceef517eca3e03c1cceb22030a3e39cb399ac86bff4e426d4fc6ae49052cc60"},
-    {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:281af09f488903fde97923c7744bb001a9b23b039a909460d0f14edc7bf59706"},
-    {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:52f2dffc8acaba9a2f27174c41c9e57f60b907bb9f096b36b1a1f3be71c6284d"},
-    {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b41156839806aecb3641f3208c0dafd3ac7775b9c4c422d82ee2a45c34ba81ca"},
-    {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d5e3fc56f88cc98ef8139255cf8cd63eb2c586531e43310ff859d6bb3a6b51f1"},
-    {file = "multidict-6.0.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8316a77808c501004802f9beebde51c9f857054a0c871bd6da8280e718444449"},
-    {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f70b98cd94886b49d91170ef23ec5c0e8ebb6f242d734ed7ed677b24d50c82cf"},
-    {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bf6774e60d67a9efe02b3616fee22441d86fab4c6d335f9d2051d19d90a40063"},
-    {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:e69924bfcdda39b722ef4d9aa762b2dd38e4632b3641b1d9a57ca9cd18f2f83a"},
-    {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:6b181d8c23da913d4ff585afd1155a0e1194c0b50c54fcfe286f70cdaf2b7176"},
-    {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:52509b5be062d9eafc8170e53026fbc54cf3b32759a23d07fd935fb04fc22d95"},
-    {file = "multidict-6.0.4-cp39-cp39-win32.whl", hash = "sha256:27c523fbfbdfd19c6867af7346332b62b586eed663887392cff78d614f9ec313"},
-    {file = "multidict-6.0.4-cp39-cp39-win_amd64.whl", hash = "sha256:33029f5734336aa0d4c0384525da0387ef89148dc7191aae00ca5fb23d7aafc2"},
-    {file = "multidict-6.0.4.tar.gz", hash = "sha256:3666906492efb76453c0e7b97f2cf459b0682e7402c0489a95484965dbc1da49"},
+    {file = "multidict-6.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3380252550e372e8511d49481bd836264c009adb826b23fefcc5dd3c69692f60"},
+    {file = "multidict-6.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:99f826cbf970077383d7de805c0681799491cb939c25450b9b5b3ced03ca99f1"},
+    {file = "multidict-6.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a114d03b938376557927ab23f1e950827c3b893ccb94b62fd95d430fd0e5cf53"},
+    {file = "multidict-6.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b1c416351ee6271b2f49b56ad7f308072f6f44b37118d69c2cad94f3fa8a40d5"},
+    {file = "multidict-6.1.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6b5d83030255983181005e6cfbac1617ce9746b219bc2aad52201ad121226581"},
+    {file = "multidict-6.1.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3e97b5e938051226dc025ec80980c285b053ffb1e25a3db2a3aa3bc046bf7f56"},
+    {file = "multidict-6.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d618649d4e70ac6efcbba75be98b26ef5078faad23592f9b51ca492953012429"},
+    {file = "multidict-6.1.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:10524ebd769727ac77ef2278390fb0068d83f3acb7773792a5080f2b0abf7748"},
+    {file = "multidict-6.1.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:ff3827aef427c89a25cc96ded1759271a93603aba9fb977a6d264648ebf989db"},
+    {file = "multidict-6.1.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:06809f4f0f7ab7ea2cabf9caca7d79c22c0758b58a71f9d32943ae13c7ace056"},
+    {file = "multidict-6.1.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:f179dee3b863ab1c59580ff60f9d99f632f34ccb38bf67a33ec6b3ecadd0fd76"},
+    {file = "multidict-6.1.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:aaed8b0562be4a0876ee3b6946f6869b7bcdb571a5d1496683505944e268b160"},
+    {file = "multidict-6.1.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3c8b88a2ccf5493b6c8da9076fb151ba106960a2df90c2633f342f120751a9e7"},
+    {file = "multidict-6.1.0-cp310-cp310-win32.whl", hash = "sha256:4a9cb68166a34117d6646c0023c7b759bf197bee5ad4272f420a0141d7eb03a0"},
+    {file = "multidict-6.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:20b9b5fbe0b88d0bdef2012ef7dee867f874b72528cf1d08f1d59b0e3850129d"},
+    {file = "multidict-6.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:3efe2c2cb5763f2f1b275ad2bf7a287d3f7ebbef35648a9726e3b69284a4f3d6"},
+    {file = "multidict-6.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c7053d3b0353a8b9de430a4f4b4268ac9a4fb3481af37dfe49825bf45ca24156"},
+    {file = "multidict-6.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:27e5fc84ccef8dfaabb09d82b7d179c7cf1a3fbc8a966f8274fcb4ab2eb4cadb"},
+    {file = "multidict-6.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0e2b90b43e696f25c62656389d32236e049568b39320e2735d51f08fd362761b"},
+    {file = "multidict-6.1.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d83a047959d38a7ff552ff94be767b7fd79b831ad1cd9920662db05fec24fe72"},
+    {file = "multidict-6.1.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d1a9dd711d0877a1ece3d2e4fea11a8e75741ca21954c919406b44e7cf971304"},
+    {file = "multidict-6.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec2abea24d98246b94913b76a125e855eb5c434f7c46546046372fe60f666351"},
+    {file = "multidict-6.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4867cafcbc6585e4b678876c489b9273b13e9fff9f6d6d66add5e15d11d926cb"},
+    {file = "multidict-6.1.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:5b48204e8d955c47c55b72779802b219a39acc3ee3d0116d5080c388970b76e3"},
+    {file = "multidict-6.1.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:d8fff389528cad1618fb4b26b95550327495462cd745d879a8c7c2115248e399"},
+    {file = "multidict-6.1.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a7a9541cd308eed5e30318430a9c74d2132e9a8cb46b901326272d780bf2d423"},
+    {file = "multidict-6.1.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:da1758c76f50c39a2efd5e9859ce7d776317eb1dd34317c8152ac9251fc574a3"},
+    {file = "multidict-6.1.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c943a53e9186688b45b323602298ab727d8865d8c9ee0b17f8d62d14b56f0753"},
+    {file = "multidict-6.1.0-cp311-cp311-win32.whl", hash = "sha256:90f8717cb649eea3504091e640a1b8568faad18bd4b9fcd692853a04475a4b80"},
+    {file = "multidict-6.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:82176036e65644a6cc5bd619f65f6f19781e8ec2e5330f51aa9ada7504cc1926"},
+    {file = "multidict-6.1.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:b04772ed465fa3cc947db808fa306d79b43e896beb677a56fb2347ca1a49c1fa"},
+    {file = "multidict-6.1.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:6180c0ae073bddeb5a97a38c03f30c233e0a4d39cd86166251617d1bbd0af436"},
+    {file = "multidict-6.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:071120490b47aa997cca00666923a83f02c7fbb44f71cf7f136df753f7fa8761"},
+    {file = "multidict-6.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50b3a2710631848991d0bf7de077502e8994c804bb805aeb2925a981de58ec2e"},
+    {file = "multidict-6.1.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b58c621844d55e71c1b7f7c498ce5aa6985d743a1a59034c57a905b3f153c1ef"},
+    {file = "multidict-6.1.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:55b6d90641869892caa9ca42ff913f7ff1c5ece06474fbd32fb2cf6834726c95"},
+    {file = "multidict-6.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b820514bfc0b98a30e3d85462084779900347e4d49267f747ff54060cc33925"},
+    {file = "multidict-6.1.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:10a9b09aba0c5b48c53761b7c720aaaf7cf236d5fe394cd399c7ba662d5f9966"},
+    {file = "multidict-6.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1e16bf3e5fc9f44632affb159d30a437bfe286ce9e02754759be5536b169b305"},
+    {file = "multidict-6.1.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:76f364861c3bfc98cbbcbd402d83454ed9e01a5224bb3a28bf70002a230f73e2"},
+    {file = "multidict-6.1.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:820c661588bd01a0aa62a1283f20d2be4281b086f80dad9e955e690c75fb54a2"},
+    {file = "multidict-6.1.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:0e5f362e895bc5b9e67fe6e4ded2492d8124bdf817827f33c5b46c2fe3ffaca6"},
+    {file = "multidict-6.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3ec660d19bbc671e3a6443325f07263be452c453ac9e512f5eb935e7d4ac28b3"},
+    {file = "multidict-6.1.0-cp312-cp312-win32.whl", hash = "sha256:58130ecf8f7b8112cdb841486404f1282b9c86ccb30d3519faf301b2e5659133"},
+    {file = "multidict-6.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:188215fc0aafb8e03341995e7c4797860181562380f81ed0a87ff455b70bf1f1"},
+    {file = "multidict-6.1.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:d569388c381b24671589335a3be6e1d45546c2988c2ebe30fdcada8457a31008"},
+    {file = "multidict-6.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:052e10d2d37810b99cc170b785945421141bf7bb7d2f8799d431e7db229c385f"},
+    {file = "multidict-6.1.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f90c822a402cb865e396a504f9fc8173ef34212a342d92e362ca498cad308e28"},
+    {file = "multidict-6.1.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b225d95519a5bf73860323e633a664b0d85ad3d5bede6d30d95b35d4dfe8805b"},
+    {file = "multidict-6.1.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:23bfd518810af7de1116313ebd9092cb9aa629beb12f6ed631ad53356ed6b86c"},
+    {file = "multidict-6.1.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5c09fcfdccdd0b57867577b719c69e347a436b86cd83747f179dbf0cc0d4c1f3"},
+    {file = "multidict-6.1.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf6bea52ec97e95560af5ae576bdac3aa3aae0b6758c6efa115236d9e07dae44"},
+    {file = "multidict-6.1.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:57feec87371dbb3520da6192213c7d6fc892d5589a93db548331954de8248fd2"},
+    {file = "multidict-6.1.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:0c3f390dc53279cbc8ba976e5f8035eab997829066756d811616b652b00a23a3"},
+    {file = "multidict-6.1.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:59bfeae4b25ec05b34f1956eaa1cb38032282cd4dfabc5056d0a1ec4d696d3aa"},
+    {file = "multidict-6.1.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b2f59caeaf7632cc633b5cf6fc449372b83bbdf0da4ae04d5be36118e46cc0aa"},
+    {file = "multidict-6.1.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:37bb93b2178e02b7b618893990941900fd25b6b9ac0fa49931a40aecdf083fe4"},
+    {file = "multidict-6.1.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4e9f48f58c2c523d5a06faea47866cd35b32655c46b443f163d08c6d0ddb17d6"},
+    {file = "multidict-6.1.0-cp313-cp313-win32.whl", hash = "sha256:3a37ffb35399029b45c6cc33640a92bef403c9fd388acce75cdc88f58bd19a81"},
+    {file = "multidict-6.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:e9aa71e15d9d9beaad2c6b9319edcdc0a49a43ef5c0a4c8265ca9ee7d6c67774"},
+    {file = "multidict-6.1.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:db7457bac39421addd0c8449933ac32d8042aae84a14911a757ae6ca3eef1392"},
+    {file = "multidict-6.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d094ddec350a2fb899fec68d8353c78233debde9b7d8b4beeafa70825f1c281a"},
+    {file = "multidict-6.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:5845c1fd4866bb5dd3125d89b90e57ed3138241540897de748cdf19de8a2fca2"},
+    {file = "multidict-6.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9079dfc6a70abe341f521f78405b8949f96db48da98aeb43f9907f342f627cdc"},
+    {file = "multidict-6.1.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3914f5aaa0f36d5d60e8ece6a308ee1c9784cd75ec8151062614657a114c4478"},
+    {file = "multidict-6.1.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c08be4f460903e5a9d0f76818db3250f12e9c344e79314d1d570fc69d7f4eae4"},
+    {file = "multidict-6.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d093be959277cb7dee84b801eb1af388b6ad3ca6a6b6bf1ed7585895789d027d"},
+    {file = "multidict-6.1.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3702ea6872c5a2a4eeefa6ffd36b042e9773f05b1f37ae3ef7264b1163c2dcf6"},
+    {file = "multidict-6.1.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:2090f6a85cafc5b2db085124d752757c9d251548cedabe9bd31afe6363e0aff2"},
+    {file = "multidict-6.1.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:f67f217af4b1ff66c68a87318012de788dd95fcfeb24cc889011f4e1c7454dfd"},
+    {file = "multidict-6.1.0-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:189f652a87e876098bbc67b4da1049afb5f5dfbaa310dd67c594b01c10388db6"},
+    {file = "multidict-6.1.0-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:6bb5992037f7a9eff7991ebe4273ea7f51f1c1c511e6a2ce511d0e7bdb754492"},
+    {file = "multidict-6.1.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:ac10f4c2b9e770c4e393876e35a7046879d195cd123b4f116d299d442b335bcd"},
+    {file = "multidict-6.1.0-cp38-cp38-win32.whl", hash = "sha256:e27bbb6d14416713a8bd7aaa1313c0fc8d44ee48d74497a0ff4c3a1b6ccb5167"},
+    {file = "multidict-6.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:22f3105d4fb15c8f57ff3959a58fcab6ce36814486500cd7485651230ad4d4ef"},
+    {file = "multidict-6.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:4e18b656c5e844539d506a0a06432274d7bd52a7487e6828c63a63d69185626c"},
+    {file = "multidict-6.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a185f876e69897a6f3325c3f19f26a297fa058c5e456bfcff8015e9a27e83ae1"},
+    {file = "multidict-6.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ab7c4ceb38d91570a650dba194e1ca87c2b543488fe9309b4212694174fd539c"},
+    {file = "multidict-6.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e617fb6b0b6953fffd762669610c1c4ffd05632c138d61ac7e14ad187870669c"},
+    {file = "multidict-6.1.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:16e5f4bf4e603eb1fdd5d8180f1a25f30056f22e55ce51fb3d6ad4ab29f7d96f"},
+    {file = "multidict-6.1.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f4c035da3f544b1882bac24115f3e2e8760f10a0107614fc9839fd232200b875"},
+    {file = "multidict-6.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:957cf8e4b6e123a9eea554fa7ebc85674674b713551de587eb318a2df3e00255"},
+    {file = "multidict-6.1.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:483a6aea59cb89904e1ceabd2b47368b5600fb7de78a6e4a2c2987b2d256cf30"},
+    {file = "multidict-6.1.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:87701f25a2352e5bf7454caa64757642734da9f6b11384c1f9d1a8e699758057"},
+    {file = "multidict-6.1.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:682b987361e5fd7a139ed565e30d81fd81e9629acc7d925a205366877d8c8657"},
+    {file = "multidict-6.1.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:ce2186a7df133a9c895dea3331ddc5ddad42cdd0d1ea2f0a51e5d161e4762f28"},
+    {file = "multidict-6.1.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:9f636b730f7e8cb19feb87094949ba54ee5357440b9658b2a32a5ce4bce53972"},
+    {file = "multidict-6.1.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:73eae06aa53af2ea5270cc066dcaf02cc60d2994bbb2c4ef5764949257d10f43"},
+    {file = "multidict-6.1.0-cp39-cp39-win32.whl", hash = "sha256:1ca0083e80e791cffc6efce7660ad24af66c8d4079d2a750b29001b53ff59ada"},
+    {file = "multidict-6.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:aa466da5b15ccea564bdab9c89175c762bc12825f4659c11227f515cee76fa4a"},
+    {file = "multidict-6.1.0-py3-none-any.whl", hash = "sha256:48e171e52d1c4d33888e529b999e5900356b9ae588c2f09a52dcefb158b27506"},
+    {file = "multidict-6.1.0.tar.gz", hash = "sha256:22ae2ebf9b0c69d206c003e2f6a914ea33f0a932d4aa16f236afc049d9958f4a"},
+]
+
+[package.dependencies]
+typing-extensions = {version = ">=4.1.0", markers = "python_version < \"3.11\""}
+
+[[package]]
+name = "numpy"
+version = "1.26.4"
+description = "Fundamental package for array computing in Python"
+optional = false
+python-versions = ">=3.9"
+files = [
+    {file = "numpy-1.26.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0"},
+    {file = "numpy-1.26.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a"},
+    {file = "numpy-1.26.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d209d8969599b27ad20994c8e41936ee0964e6da07478d6c35016bc386b66ad4"},
+    {file = "numpy-1.26.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f"},
+    {file = "numpy-1.26.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:62b8e4b1e28009ef2846b4c7852046736bab361f7aeadeb6a5b89ebec3c7055a"},
+    {file = "numpy-1.26.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a4abb4f9001ad2858e7ac189089c42178fcce737e4169dc61321660f1a96c7d2"},
+    {file = "numpy-1.26.4-cp310-cp310-win32.whl", hash = "sha256:bfe25acf8b437eb2a8b2d49d443800a5f18508cd811fea3181723922a8a82b07"},
+    {file = "numpy-1.26.4-cp310-cp310-win_amd64.whl", hash = "sha256:b97fe8060236edf3662adfc2c633f56a08ae30560c56310562cb4f95500022d5"},
+    {file = "numpy-1.26.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4c66707fabe114439db9068ee468c26bbdf909cac0fb58686a42a24de1760c71"},
+    {file = "numpy-1.26.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:edd8b5fe47dab091176d21bb6de568acdd906d1887a4584a15a9a96a1dca06ef"},
+    {file = "numpy-1.26.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ab55401287bfec946ced39700c053796e7cc0e3acbef09993a9ad2adba6ca6e"},
+    {file = "numpy-1.26.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:666dbfb6ec68962c033a450943ded891bed2d54e6755e35e5835d63f4f6931d5"},
+    {file = "numpy-1.26.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:96ff0b2ad353d8f990b63294c8986f1ec3cb19d749234014f4e7eb0112ceba5a"},
+    {file = "numpy-1.26.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:60dedbb91afcbfdc9bc0b1f3f402804070deed7392c23eb7a7f07fa857868e8a"},
+    {file = "numpy-1.26.4-cp311-cp311-win32.whl", hash = "sha256:1af303d6b2210eb850fcf03064d364652b7120803a0b872f5211f5234b399f20"},
+    {file = "numpy-1.26.4-cp311-cp311-win_amd64.whl", hash = "sha256:cd25bcecc4974d09257ffcd1f098ee778f7834c3ad767fe5db785be9a4aa9cb2"},
+    {file = "numpy-1.26.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218"},
+    {file = "numpy-1.26.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b"},
+    {file = "numpy-1.26.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b"},
+    {file = "numpy-1.26.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed"},
+    {file = "numpy-1.26.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a"},
+    {file = "numpy-1.26.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0"},
+    {file = "numpy-1.26.4-cp312-cp312-win32.whl", hash = "sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110"},
+    {file = "numpy-1.26.4-cp312-cp312-win_amd64.whl", hash = "sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818"},
+    {file = "numpy-1.26.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7349ab0fa0c429c82442a27a9673fc802ffdb7c7775fad780226cb234965e53c"},
+    {file = "numpy-1.26.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:52b8b60467cd7dd1e9ed082188b4e6bb35aa5cdd01777621a1658910745b90be"},
+    {file = "numpy-1.26.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d5241e0a80d808d70546c697135da2c613f30e28251ff8307eb72ba696945764"},
+    {file = "numpy-1.26.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f870204a840a60da0b12273ef34f7051e98c3b5961b61b0c2c1be6dfd64fbcd3"},
+    {file = "numpy-1.26.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:679b0076f67ecc0138fd2ede3a8fd196dddc2ad3254069bcb9faf9a79b1cebcd"},
+    {file = "numpy-1.26.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:47711010ad8555514b434df65f7d7b076bb8261df1ca9bb78f53d3b2db02e95c"},
+    {file = "numpy-1.26.4-cp39-cp39-win32.whl", hash = "sha256:a354325ee03388678242a4d7ebcd08b5c727033fcff3b2f536aea978e15ee9e6"},
+    {file = "numpy-1.26.4-cp39-cp39-win_amd64.whl", hash = "sha256:3373d5d70a5fe74a2c1bb6d2cfd9609ecf686d47a2d7b1d37a8f3b6bf6003aea"},
+    {file = "numpy-1.26.4-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:afedb719a9dcfc7eaf2287b839d8198e06dcd4cb5d276a3df279231138e83d30"},
+    {file = "numpy-1.26.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95a7476c59002f2f6c590b9b7b998306fba6a5aa646b1e22ddfeaf8f78c3a29c"},
+    {file = "numpy-1.26.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7e50d0a0cc3189f9cb0aeb3a6a6af18c16f59f004b866cd2be1c14b36134a4a0"},
+    {file = "numpy-1.26.4.tar.gz", hash = "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010"},
 ]
 
 [[package]]
 name = "packaging"
-version = "23.1"
+version = "24.1"
 description = "Core utilities for Python packages"
 optional = false
-python-versions = ">=3.7"
+python-versions = ">=3.8"
 files = [
-    {file = "packaging-23.1-py3-none-any.whl", hash = "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61"},
-    {file = "packaging-23.1.tar.gz", hash = "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"},
+    {file = "packaging-24.1-py3-none-any.whl", hash = "sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124"},
+    {file = "packaging-24.1.tar.gz", hash = "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002"},
 ]
 
 [[package]]
 name = "pluggy"
-version = "1.3.0"
+version = "1.5.0"
 description = "plugin and hook calling mechanisms for python"
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "pluggy-1.3.0-py3-none-any.whl", hash = "sha256:d89c696a773f8bd377d18e5ecda92b7a3793cbe66c87060a6fb58c7b6e1061f7"},
-    {file = "pluggy-1.3.0.tar.gz", hash = "sha256:cf61ae8f126ac6f7c451172cf30e3e43d3ca77615509771b3a984a0730651e12"},
+    {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"},
+    {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"},
 ]
 
 [package.extras]
@@ -601,109 +709,120 @@ testing = ["pytest", "pytest-benchmark"]
 
 [[package]]
 name = "pydantic"
-version = "2.6.4"
+version = "2.9.1"
 description = "Data validation using Python type hints"
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "pydantic-2.6.4-py3-none-any.whl", hash = "sha256:cc46fce86607580867bdc3361ad462bab9c222ef042d3da86f2fb333e1d916c5"},
-    {file = "pydantic-2.6.4.tar.gz", hash = "sha256:b1704e0847db01817624a6b86766967f552dd9dbf3afba4004409f908dcc84e6"},
+    {file = "pydantic-2.9.1-py3-none-any.whl", hash = "sha256:7aff4db5fdf3cf573d4b3c30926a510a10e19a0774d38fc4967f78beb6deb612"},
+    {file = "pydantic-2.9.1.tar.gz", hash = "sha256:1363c7d975c7036df0db2b4a61f2e062fbc0aa5ab5f2772e0ffc7191a4f4bce2"},
 ]
 
 [package.dependencies]
-annotated-types = ">=0.4.0"
-pydantic-core = "2.16.3"
-typing-extensions = ">=4.6.1"
+annotated-types = ">=0.6.0"
+pydantic-core = "2.23.3"
+typing-extensions = {version = ">=4.6.1", markers = "python_version < \"3.13\""}
 
 [package.extras]
 email = ["email-validator (>=2.0.0)"]
+timezone = ["tzdata"]
 
 [[package]]
 name = "pydantic-core"
-version = "2.16.3"
-description = ""
+version = "2.23.3"
+description = "Core functionality for Pydantic validation and serialization"
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "pydantic_core-2.16.3-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:75b81e678d1c1ede0785c7f46690621e4c6e63ccd9192af1f0bd9d504bbb6bf4"},
-    {file = "pydantic_core-2.16.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9c865a7ee6f93783bd5d781af5a4c43dadc37053a5b42f7d18dc019f8c9d2bd1"},
-    {file = "pydantic_core-2.16.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:162e498303d2b1c036b957a1278fa0899d02b2842f1ff901b6395104c5554a45"},
-    {file = "pydantic_core-2.16.3-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2f583bd01bbfbff4eaee0868e6fc607efdfcc2b03c1c766b06a707abbc856187"},
-    {file = "pydantic_core-2.16.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b926dd38db1519ed3043a4de50214e0d600d404099c3392f098a7f9d75029ff8"},
-    {file = "pydantic_core-2.16.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:716b542728d4c742353448765aa7cdaa519a7b82f9564130e2b3f6766018c9ec"},
-    {file = "pydantic_core-2.16.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc4ad7f7ee1a13d9cb49d8198cd7d7e3aa93e425f371a68235f784e99741561f"},
-    {file = "pydantic_core-2.16.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:bd87f48924f360e5d1c5f770d6155ce0e7d83f7b4e10c2f9ec001c73cf475c99"},
-    {file = "pydantic_core-2.16.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:0df446663464884297c793874573549229f9eca73b59360878f382a0fc085979"},
-    {file = "pydantic_core-2.16.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:4df8a199d9f6afc5ae9a65f8f95ee52cae389a8c6b20163762bde0426275b7db"},
-    {file = "pydantic_core-2.16.3-cp310-none-win32.whl", hash = "sha256:456855f57b413f077dff513a5a28ed838dbbb15082ba00f80750377eed23d132"},
-    {file = "pydantic_core-2.16.3-cp310-none-win_amd64.whl", hash = "sha256:732da3243e1b8d3eab8c6ae23ae6a58548849d2e4a4e03a1924c8ddf71a387cb"},
-    {file = "pydantic_core-2.16.3-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:519ae0312616026bf4cedc0fe459e982734f3ca82ee8c7246c19b650b60a5ee4"},
-    {file = "pydantic_core-2.16.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b3992a322a5617ded0a9f23fd06dbc1e4bd7cf39bc4ccf344b10f80af58beacd"},
-    {file = "pydantic_core-2.16.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8d62da299c6ecb04df729e4b5c52dc0d53f4f8430b4492b93aa8de1f541c4aac"},
-    {file = "pydantic_core-2.16.3-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2acca2be4bb2f2147ada8cac612f8a98fc09f41c89f87add7256ad27332c2fda"},
-    {file = "pydantic_core-2.16.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1b662180108c55dfbf1280d865b2d116633d436cfc0bba82323554873967b340"},
-    {file = "pydantic_core-2.16.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e7c6ed0dc9d8e65f24f5824291550139fe6f37fac03788d4580da0d33bc00c97"},
-    {file = "pydantic_core-2.16.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a6b1bb0827f56654b4437955555dc3aeeebeddc47c2d7ed575477f082622c49e"},
-    {file = "pydantic_core-2.16.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e56f8186d6210ac7ece503193ec84104da7ceb98f68ce18c07282fcc2452e76f"},
-    {file = "pydantic_core-2.16.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:936e5db01dd49476fa8f4383c259b8b1303d5dd5fb34c97de194560698cc2c5e"},
-    {file = "pydantic_core-2.16.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:33809aebac276089b78db106ee692bdc9044710e26f24a9a2eaa35a0f9fa70ba"},
-    {file = "pydantic_core-2.16.3-cp311-none-win32.whl", hash = "sha256:ded1c35f15c9dea16ead9bffcde9bb5c7c031bff076355dc58dcb1cb436c4721"},
-    {file = "pydantic_core-2.16.3-cp311-none-win_amd64.whl", hash = "sha256:d89ca19cdd0dd5f31606a9329e309d4fcbb3df860960acec32630297d61820df"},
-    {file = "pydantic_core-2.16.3-cp311-none-win_arm64.whl", hash = "sha256:6162f8d2dc27ba21027f261e4fa26f8bcb3cf9784b7f9499466a311ac284b5b9"},
-    {file = "pydantic_core-2.16.3-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:0f56ae86b60ea987ae8bcd6654a887238fd53d1384f9b222ac457070b7ac4cff"},
-    {file = "pydantic_core-2.16.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c9bd22a2a639e26171068f8ebb5400ce2c1bc7d17959f60a3b753ae13c632975"},
-    {file = "pydantic_core-2.16.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4204e773b4b408062960e65468d5346bdfe139247ee5f1ca2a378983e11388a2"},
-    {file = "pydantic_core-2.16.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f651dd19363c632f4abe3480a7c87a9773be27cfe1341aef06e8759599454120"},
-    {file = "pydantic_core-2.16.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aaf09e615a0bf98d406657e0008e4a8701b11481840be7d31755dc9f97c44053"},
-    {file = "pydantic_core-2.16.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8e47755d8152c1ab5b55928ab422a76e2e7b22b5ed8e90a7d584268dd49e9c6b"},
-    {file = "pydantic_core-2.16.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:500960cb3a0543a724a81ba859da816e8cf01b0e6aaeedf2c3775d12ee49cade"},
-    {file = "pydantic_core-2.16.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:cf6204fe865da605285c34cf1172879d0314ff267b1c35ff59de7154f35fdc2e"},
-    {file = "pydantic_core-2.16.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d33dd21f572545649f90c38c227cc8631268ba25c460b5569abebdd0ec5974ca"},
-    {file = "pydantic_core-2.16.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:49d5d58abd4b83fb8ce763be7794d09b2f50f10aa65c0f0c1696c677edeb7cbf"},
-    {file = "pydantic_core-2.16.3-cp312-none-win32.whl", hash = "sha256:f53aace168a2a10582e570b7736cc5bef12cae9cf21775e3eafac597e8551fbe"},
-    {file = "pydantic_core-2.16.3-cp312-none-win_amd64.whl", hash = "sha256:0d32576b1de5a30d9a97f300cc6a3f4694c428d956adbc7e6e2f9cad279e45ed"},
-    {file = "pydantic_core-2.16.3-cp312-none-win_arm64.whl", hash = "sha256:ec08be75bb268473677edb83ba71e7e74b43c008e4a7b1907c6d57e940bf34b6"},
-    {file = "pydantic_core-2.16.3-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:b1f6f5938d63c6139860f044e2538baeee6f0b251a1816e7adb6cbce106a1f01"},
-    {file = "pydantic_core-2.16.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2a1ef6a36fdbf71538142ed604ad19b82f67b05749512e47f247a6ddd06afdc7"},
-    {file = "pydantic_core-2.16.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:704d35ecc7e9c31d48926150afada60401c55efa3b46cd1ded5a01bdffaf1d48"},
-    {file = "pydantic_core-2.16.3-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d937653a696465677ed583124b94a4b2d79f5e30b2c46115a68e482c6a591c8a"},
-    {file = "pydantic_core-2.16.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c9803edf8e29bd825f43481f19c37f50d2b01899448273b3a7758441b512acf8"},
-    {file = "pydantic_core-2.16.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:72282ad4892a9fb2da25defeac8c2e84352c108705c972db82ab121d15f14e6d"},
-    {file = "pydantic_core-2.16.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f752826b5b8361193df55afcdf8ca6a57d0232653494ba473630a83ba50d8c9"},
-    {file = "pydantic_core-2.16.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4384a8f68ddb31a0b0c3deae88765f5868a1b9148939c3f4121233314ad5532c"},
-    {file = "pydantic_core-2.16.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:a4b2bf78342c40b3dc830880106f54328928ff03e357935ad26c7128bbd66ce8"},
-    {file = "pydantic_core-2.16.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:13dcc4802961b5f843a9385fc821a0b0135e8c07fc3d9949fd49627c1a5e6ae5"},
-    {file = "pydantic_core-2.16.3-cp38-none-win32.whl", hash = "sha256:e3e70c94a0c3841e6aa831edab1619ad5c511199be94d0c11ba75fe06efe107a"},
-    {file = "pydantic_core-2.16.3-cp38-none-win_amd64.whl", hash = "sha256:ecdf6bf5f578615f2e985a5e1f6572e23aa632c4bd1dc67f8f406d445ac115ed"},
-    {file = "pydantic_core-2.16.3-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:bda1ee3e08252b8d41fa5537413ffdddd58fa73107171a126d3b9ff001b9b820"},
-    {file = "pydantic_core-2.16.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:21b888c973e4f26b7a96491c0965a8a312e13be108022ee510248fe379a5fa23"},
-    {file = "pydantic_core-2.16.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:be0ec334369316fa73448cc8c982c01e5d2a81c95969d58b8f6e272884df0074"},
-    {file = "pydantic_core-2.16.3-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b5b6079cc452a7c53dd378c6f881ac528246b3ac9aae0f8eef98498a75657805"},
-    {file = "pydantic_core-2.16.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7ee8d5f878dccb6d499ba4d30d757111847b6849ae07acdd1205fffa1fc1253c"},
-    {file = "pydantic_core-2.16.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7233d65d9d651242a68801159763d09e9ec96e8a158dbf118dc090cd77a104c9"},
-    {file = "pydantic_core-2.16.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c6119dc90483a5cb50a1306adb8d52c66e447da88ea44f323e0ae1a5fcb14256"},
-    {file = "pydantic_core-2.16.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:578114bc803a4c1ff9946d977c221e4376620a46cf78da267d946397dc9514a8"},
-    {file = "pydantic_core-2.16.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d8f99b147ff3fcf6b3cc60cb0c39ea443884d5559a30b1481e92495f2310ff2b"},
-    {file = "pydantic_core-2.16.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4ac6b4ce1e7283d715c4b729d8f9dab9627586dafce81d9eaa009dd7f25dd972"},
-    {file = "pydantic_core-2.16.3-cp39-none-win32.whl", hash = "sha256:e7774b570e61cb998490c5235740d475413a1f6de823169b4cf94e2fe9e9f6b2"},
-    {file = "pydantic_core-2.16.3-cp39-none-win_amd64.whl", hash = "sha256:9091632a25b8b87b9a605ec0e61f241c456e9248bfdcf7abdf344fdb169c81cf"},
-    {file = "pydantic_core-2.16.3-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:36fa178aacbc277bc6b62a2c3da95226520da4f4e9e206fdf076484363895d2c"},
-    {file = "pydantic_core-2.16.3-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:dcca5d2bf65c6fb591fff92da03f94cd4f315972f97c21975398bd4bd046854a"},
-    {file = "pydantic_core-2.16.3-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2a72fb9963cba4cd5793854fd12f4cfee731e86df140f59ff52a49b3552db241"},
-    {file = "pydantic_core-2.16.3-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b60cc1a081f80a2105a59385b92d82278b15d80ebb3adb200542ae165cd7d183"},
-    {file = "pydantic_core-2.16.3-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:cbcc558401de90a746d02ef330c528f2e668c83350f045833543cd57ecead1ad"},
-    {file = "pydantic_core-2.16.3-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:fee427241c2d9fb7192b658190f9f5fd6dfe41e02f3c1489d2ec1e6a5ab1e04a"},
-    {file = "pydantic_core-2.16.3-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f4cb85f693044e0f71f394ff76c98ddc1bc0953e48c061725e540396d5c8a2e1"},
-    {file = "pydantic_core-2.16.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:b29eeb887aa931c2fcef5aa515d9d176d25006794610c264ddc114c053bf96fe"},
-    {file = "pydantic_core-2.16.3-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:a425479ee40ff021f8216c9d07a6a3b54b31c8267c6e17aa88b70d7ebd0e5e5b"},
-    {file = "pydantic_core-2.16.3-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:5c5cbc703168d1b7a838668998308018a2718c2130595e8e190220238addc96f"},
-    {file = "pydantic_core-2.16.3-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:99b6add4c0b39a513d323d3b93bc173dac663c27b99860dd5bf491b240d26137"},
-    {file = "pydantic_core-2.16.3-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75f76ee558751746d6a38f89d60b6228fa174e5172d143886af0f85aa306fd89"},
-    {file = "pydantic_core-2.16.3-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:00ee1c97b5364b84cb0bd82e9bbf645d5e2871fb8c58059d158412fee2d33d8a"},
-    {file = "pydantic_core-2.16.3-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:287073c66748f624be4cef893ef9174e3eb88fe0b8a78dc22e88eca4bc357ca6"},
-    {file = "pydantic_core-2.16.3-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:ed25e1835c00a332cb10c683cd39da96a719ab1dfc08427d476bce41b92531fc"},
-    {file = "pydantic_core-2.16.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:86b3d0033580bd6bbe07590152007275bd7af95f98eaa5bd36f3da219dcd93da"},
-    {file = "pydantic_core-2.16.3.tar.gz", hash = "sha256:1cac689f80a3abab2d3c0048b29eea5751114054f032a941a32de4c852c59cad"},
+    {file = "pydantic_core-2.23.3-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:7f10a5d1b9281392f1bf507d16ac720e78285dfd635b05737c3911637601bae6"},
+    {file = "pydantic_core-2.23.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3c09a7885dd33ee8c65266e5aa7fb7e2f23d49d8043f089989726391dd7350c5"},
+    {file = "pydantic_core-2.23.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6470b5a1ec4d1c2e9afe928c6cb37eb33381cab99292a708b8cb9aa89e62429b"},
+    {file = "pydantic_core-2.23.3-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9172d2088e27d9a185ea0a6c8cebe227a9139fd90295221d7d495944d2367700"},
+    {file = "pydantic_core-2.23.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:86fc6c762ca7ac8fbbdff80d61b2c59fb6b7d144aa46e2d54d9e1b7b0e780e01"},
+    {file = "pydantic_core-2.23.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f0cb80fd5c2df4898693aa841425ea1727b1b6d2167448253077d2a49003e0ed"},
+    {file = "pydantic_core-2.23.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:03667cec5daf43ac4995cefa8aaf58f99de036204a37b889c24a80927b629cec"},
+    {file = "pydantic_core-2.23.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:047531242f8e9c2db733599f1c612925de095e93c9cc0e599e96cf536aaf56ba"},
+    {file = "pydantic_core-2.23.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:5499798317fff7f25dbef9347f4451b91ac2a4330c6669821c8202fd354c7bee"},
+    {file = "pydantic_core-2.23.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bbb5e45eab7624440516ee3722a3044b83fff4c0372efe183fd6ba678ff681fe"},
+    {file = "pydantic_core-2.23.3-cp310-none-win32.whl", hash = "sha256:8b5b3ed73abb147704a6e9f556d8c5cb078f8c095be4588e669d315e0d11893b"},
+    {file = "pydantic_core-2.23.3-cp310-none-win_amd64.whl", hash = "sha256:2b603cde285322758a0279995b5796d64b63060bfbe214b50a3ca23b5cee3e83"},
+    {file = "pydantic_core-2.23.3-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:c889fd87e1f1bbeb877c2ee56b63bb297de4636661cc9bbfcf4b34e5e925bc27"},
+    {file = "pydantic_core-2.23.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ea85bda3189fb27503af4c45273735bcde3dd31c1ab17d11f37b04877859ef45"},
+    {file = "pydantic_core-2.23.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a7f7f72f721223f33d3dc98a791666ebc6a91fa023ce63733709f4894a7dc611"},
+    {file = "pydantic_core-2.23.3-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2b2b55b0448e9da68f56b696f313949cda1039e8ec7b5d294285335b53104b61"},
+    {file = "pydantic_core-2.23.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c24574c7e92e2c56379706b9a3f07c1e0c7f2f87a41b6ee86653100c4ce343e5"},
+    {file = "pydantic_core-2.23.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f2b05e6ccbee333a8f4b8f4d7c244fdb7a979e90977ad9c51ea31261e2085ce0"},
+    {file = "pydantic_core-2.23.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e2c409ce1c219c091e47cb03feb3c4ed8c2b8e004efc940da0166aaee8f9d6c8"},
+    {file = "pydantic_core-2.23.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d965e8b325f443ed3196db890d85dfebbb09f7384486a77461347f4adb1fa7f8"},
+    {file = "pydantic_core-2.23.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:f56af3a420fb1ffaf43ece3ea09c2d27c444e7c40dcb7c6e7cf57aae764f2b48"},
+    {file = "pydantic_core-2.23.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5b01a078dd4f9a52494370af21aa52964e0a96d4862ac64ff7cea06e0f12d2c5"},
+    {file = "pydantic_core-2.23.3-cp311-none-win32.whl", hash = "sha256:560e32f0df04ac69b3dd818f71339983f6d1f70eb99d4d1f8e9705fb6c34a5c1"},
+    {file = "pydantic_core-2.23.3-cp311-none-win_amd64.whl", hash = "sha256:c744fa100fdea0d000d8bcddee95213d2de2e95b9c12be083370b2072333a0fa"},
+    {file = "pydantic_core-2.23.3-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:e0ec50663feedf64d21bad0809f5857bac1ce91deded203efc4a84b31b2e4305"},
+    {file = "pydantic_core-2.23.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:db6e6afcb95edbe6b357786684b71008499836e91f2a4a1e55b840955b341dbb"},
+    {file = "pydantic_core-2.23.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:98ccd69edcf49f0875d86942f4418a4e83eb3047f20eb897bffa62a5d419c8fa"},
+    {file = "pydantic_core-2.23.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a678c1ac5c5ec5685af0133262103defb427114e62eafeda12f1357a12140162"},
+    {file = "pydantic_core-2.23.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:01491d8b4d8db9f3391d93b0df60701e644ff0894352947f31fff3e52bd5c801"},
+    {file = "pydantic_core-2.23.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fcf31facf2796a2d3b7fe338fe8640aa0166e4e55b4cb108dbfd1058049bf4cb"},
+    {file = "pydantic_core-2.23.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7200fd561fb3be06827340da066df4311d0b6b8eb0c2116a110be5245dceb326"},
+    {file = "pydantic_core-2.23.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:dc1636770a809dee2bd44dd74b89cc80eb41172bcad8af75dd0bc182c2666d4c"},
+    {file = "pydantic_core-2.23.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:67a5def279309f2e23014b608c4150b0c2d323bd7bccd27ff07b001c12c2415c"},
+    {file = "pydantic_core-2.23.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:748bdf985014c6dd3e1e4cc3db90f1c3ecc7246ff5a3cd4ddab20c768b2f1dab"},
+    {file = "pydantic_core-2.23.3-cp312-none-win32.whl", hash = "sha256:255ec6dcb899c115f1e2a64bc9ebc24cc0e3ab097775755244f77360d1f3c06c"},
+    {file = "pydantic_core-2.23.3-cp312-none-win_amd64.whl", hash = "sha256:40b8441be16c1e940abebed83cd006ddb9e3737a279e339dbd6d31578b802f7b"},
+    {file = "pydantic_core-2.23.3-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:6daaf5b1ba1369a22c8b050b643250e3e5efc6a78366d323294aee54953a4d5f"},
+    {file = "pydantic_core-2.23.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d015e63b985a78a3d4ccffd3bdf22b7c20b3bbd4b8227809b3e8e75bc37f9cb2"},
+    {file = "pydantic_core-2.23.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a3fc572d9b5b5cfe13f8e8a6e26271d5d13f80173724b738557a8c7f3a8a3791"},
+    {file = "pydantic_core-2.23.3-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f6bd91345b5163ee7448bee201ed7dd601ca24f43f439109b0212e296eb5b423"},
+    {file = "pydantic_core-2.23.3-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fc379c73fd66606628b866f661e8785088afe2adaba78e6bbe80796baf708a63"},
+    {file = "pydantic_core-2.23.3-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fbdce4b47592f9e296e19ac31667daed8753c8367ebb34b9a9bd89dacaa299c9"},
+    {file = "pydantic_core-2.23.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc3cf31edf405a161a0adad83246568647c54404739b614b1ff43dad2b02e6d5"},
+    {file = "pydantic_core-2.23.3-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8e22b477bf90db71c156f89a55bfe4d25177b81fce4aa09294d9e805eec13855"},
+    {file = "pydantic_core-2.23.3-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:0a0137ddf462575d9bce863c4c95bac3493ba8e22f8c28ca94634b4a1d3e2bb4"},
+    {file = "pydantic_core-2.23.3-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:203171e48946c3164fe7691fc349c79241ff8f28306abd4cad5f4f75ed80bc8d"},
+    {file = "pydantic_core-2.23.3-cp313-none-win32.whl", hash = "sha256:76bdab0de4acb3f119c2a4bff740e0c7dc2e6de7692774620f7452ce11ca76c8"},
+    {file = "pydantic_core-2.23.3-cp313-none-win_amd64.whl", hash = "sha256:37ba321ac2a46100c578a92e9a6aa33afe9ec99ffa084424291d84e456f490c1"},
+    {file = "pydantic_core-2.23.3-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:d063c6b9fed7d992bcbebfc9133f4c24b7a7f215d6b102f3e082b1117cddb72c"},
+    {file = "pydantic_core-2.23.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:6cb968da9a0746a0cf521b2b5ef25fc5a0bee9b9a1a8214e0a1cfaea5be7e8a4"},
+    {file = "pydantic_core-2.23.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:edbefe079a520c5984e30e1f1f29325054b59534729c25b874a16a5048028d16"},
+    {file = "pydantic_core-2.23.3-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:cbaaf2ef20d282659093913da9d402108203f7cb5955020bd8d1ae5a2325d1c4"},
+    {file = "pydantic_core-2.23.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fb539d7e5dc4aac345846f290cf504d2fd3c1be26ac4e8b5e4c2b688069ff4cf"},
+    {file = "pydantic_core-2.23.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7e6f33503c5495059148cc486867e1d24ca35df5fc064686e631e314d959ad5b"},
+    {file = "pydantic_core-2.23.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:04b07490bc2f6f2717b10c3969e1b830f5720b632f8ae2f3b8b1542394c47a8e"},
+    {file = "pydantic_core-2.23.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:03795b9e8a5d7fda05f3873efc3f59105e2dcff14231680296b87b80bb327295"},
+    {file = "pydantic_core-2.23.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:c483dab0f14b8d3f0df0c6c18d70b21b086f74c87ab03c59250dbf6d3c89baba"},
+    {file = "pydantic_core-2.23.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8b2682038e255e94baf2c473dca914a7460069171ff5cdd4080be18ab8a7fd6e"},
+    {file = "pydantic_core-2.23.3-cp38-none-win32.whl", hash = "sha256:f4a57db8966b3a1d1a350012839c6a0099f0898c56512dfade8a1fe5fb278710"},
+    {file = "pydantic_core-2.23.3-cp38-none-win_amd64.whl", hash = "sha256:13dd45ba2561603681a2676ca56006d6dee94493f03d5cadc055d2055615c3ea"},
+    {file = "pydantic_core-2.23.3-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:82da2f4703894134a9f000e24965df73cc103e31e8c31906cc1ee89fde72cbd8"},
+    {file = "pydantic_core-2.23.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:dd9be0a42de08f4b58a3cc73a123f124f65c24698b95a54c1543065baca8cf0e"},
+    {file = "pydantic_core-2.23.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:89b731f25c80830c76fdb13705c68fef6a2b6dc494402987c7ea9584fe189f5d"},
+    {file = "pydantic_core-2.23.3-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c6de1ec30c4bb94f3a69c9f5f2182baeda5b809f806676675e9ef6b8dc936f28"},
+    {file = "pydantic_core-2.23.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bb68b41c3fa64587412b104294b9cbb027509dc2f6958446c502638d481525ef"},
+    {file = "pydantic_core-2.23.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1c3980f2843de5184656aab58698011b42763ccba11c4a8c35936c8dd6c7068c"},
+    {file = "pydantic_core-2.23.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94f85614f2cba13f62c3c6481716e4adeae48e1eaa7e8bac379b9d177d93947a"},
+    {file = "pydantic_core-2.23.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:510b7fb0a86dc8f10a8bb43bd2f97beb63cffad1203071dc434dac26453955cd"},
+    {file = "pydantic_core-2.23.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:1eba2f7ce3e30ee2170410e2171867ea73dbd692433b81a93758ab2de6c64835"},
+    {file = "pydantic_core-2.23.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4b259fd8409ab84b4041b7b3f24dcc41e4696f180b775961ca8142b5b21d0e70"},
+    {file = "pydantic_core-2.23.3-cp39-none-win32.whl", hash = "sha256:40d9bd259538dba2f40963286009bf7caf18b5112b19d2b55b09c14dde6db6a7"},
+    {file = "pydantic_core-2.23.3-cp39-none-win_amd64.whl", hash = "sha256:5a8cd3074a98ee70173a8633ad3c10e00dcb991ecec57263aacb4095c5efb958"},
+    {file = "pydantic_core-2.23.3-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:f399e8657c67313476a121a6944311fab377085ca7f490648c9af97fc732732d"},
+    {file = "pydantic_core-2.23.3-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:6b5547d098c76e1694ba85f05b595720d7c60d342f24d5aad32c3049131fa5c4"},
+    {file = "pydantic_core-2.23.3-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0dda0290a6f608504882d9f7650975b4651ff91c85673341789a476b1159f211"},
+    {file = "pydantic_core-2.23.3-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:65b6e5da855e9c55a0c67f4db8a492bf13d8d3316a59999cfbaf98cc6e401961"},
+    {file = "pydantic_core-2.23.3-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:09e926397f392059ce0afdcac920df29d9c833256354d0c55f1584b0b70cf07e"},
+    {file = "pydantic_core-2.23.3-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:87cfa0ed6b8c5bd6ae8b66de941cece179281239d482f363814d2b986b79cedc"},
+    {file = "pydantic_core-2.23.3-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:e61328920154b6a44d98cabcb709f10e8b74276bc709c9a513a8c37a18786cc4"},
+    {file = "pydantic_core-2.23.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:ce3317d155628301d649fe5e16a99528d5680af4ec7aa70b90b8dacd2d725c9b"},
+    {file = "pydantic_core-2.23.3-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:e89513f014c6be0d17b00a9a7c81b1c426f4eb9224b15433f3d98c1a071f8433"},
+    {file = "pydantic_core-2.23.3-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:4f62c1c953d7ee375df5eb2e44ad50ce2f5aff931723b398b8bc6f0ac159791a"},
+    {file = "pydantic_core-2.23.3-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2718443bc671c7ac331de4eef9b673063b10af32a0bb385019ad61dcf2cc8f6c"},
+    {file = "pydantic_core-2.23.3-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0d90e08b2727c5d01af1b5ef4121d2f0c99fbee692c762f4d9d0409c9da6541"},
+    {file = "pydantic_core-2.23.3-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2b676583fc459c64146debea14ba3af54e540b61762dfc0613dc4e98c3f66eeb"},
+    {file = "pydantic_core-2.23.3-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:50e4661f3337977740fdbfbae084ae5693e505ca2b3130a6d4eb0f2281dc43b8"},
+    {file = "pydantic_core-2.23.3-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:68f4cf373f0de6abfe599a38307f4417c1c867ca381c03df27c873a9069cda25"},
+    {file = "pydantic_core-2.23.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:59d52cf01854cb26c46958552a21acb10dd78a52aa34c86f284e66b209db8cab"},
+    {file = "pydantic_core-2.23.3.tar.gz", hash = "sha256:3cb0f65d8b4121c1b015c60104a685feb929a29d7cf204387c7f2688c7974690"},
 ]
 
 [package.dependencies]
@@ -711,13 +830,13 @@ typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0"
 
 [[package]]
 name = "pytest"
-version = "7.4.0"
+version = "7.4.4"
 description = "pytest: simple powerful testing with Python"
 optional = false
 python-versions = ">=3.7"
 files = [
-    {file = "pytest-7.4.0-py3-none-any.whl", hash = "sha256:78bf16451a2eb8c7a2ea98e32dc119fd2aa758f1d5d66dbf0a59d69a3969df32"},
-    {file = "pytest-7.4.0.tar.gz", hash = "sha256:b4bf8c45bd59934ed84001ad51e11b4ee40d40a1229d2c79f9c592b0a3f6bd8a"},
+    {file = "pytest-7.4.4-py3-none-any.whl", hash = "sha256:b090cdf5ed60bf4c45261be03239c2c1c22df034fbffe691abe93cd80cea01d8"},
+    {file = "pytest-7.4.4.tar.gz", hash = "sha256:2cf0005922c6ace4a3e2ec8b4080eb0d9753fdc93107415332f50ce9e7994280"},
 ]
 
 [package.dependencies]
@@ -733,13 +852,13 @@ testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "no
 
 [[package]]
 name = "pytest-asyncio"
-version = "0.21.1"
+version = "0.21.2"
 description = "Pytest support for asyncio"
 optional = false
 python-versions = ">=3.7"
 files = [
-    {file = "pytest-asyncio-0.21.1.tar.gz", hash = "sha256:40a7eae6dded22c7b604986855ea48400ab15b069ae38116e8c01238e9eeb64d"},
-    {file = "pytest_asyncio-0.21.1-py3-none-any.whl", hash = "sha256:8666c1c8ac02631d7c51ba282e0c69a8a452b211ffedf2599099845da5c5c37b"},
+    {file = "pytest_asyncio-0.21.2-py3-none-any.whl", hash = "sha256:ab664c88bb7998f711d8039cacd4884da6430886ae8bbd4eded552ed2004f16b"},
+    {file = "pytest_asyncio-0.21.2.tar.gz", hash = "sha256:d67738fc232b94b326b9d060750beb16e0074210b98dd8b58a5239fa2a154f45"},
 ]
 
 [package.dependencies]
@@ -774,73 +893,75 @@ files = [
 
 [[package]]
 name = "pyyaml"
-version = "6.0.1"
+version = "6.0.2"
 description = "YAML parser and emitter for Python"
 optional = false
-python-versions = ">=3.6"
+python-versions = ">=3.8"
 files = [
-    {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"},
-    {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"},
-    {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"},
-    {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"},
-    {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"},
-    {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"},
-    {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"},
-    {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"},
-    {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"},
-    {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"},
-    {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"},
-    {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"},
-    {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"},
-    {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"},
-    {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"},
-    {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"},
-    {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"},
-    {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"},
-    {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"},
-    {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"},
-    {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"},
-    {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"},
-    {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"},
-    {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"},
-    {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"},
-    {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"},
-    {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"},
-    {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"},
-    {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"},
-    {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"},
-    {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"},
-    {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"},
-    {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"},
-    {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"},
-    {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"},
-    {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"},
-    {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"},
-    {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"},
-    {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"},
-    {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"},
-    {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"},
-    {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"},
-    {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"},
-    {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"},
-    {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"},
-    {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"},
-    {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"},
-    {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"},
-    {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"},
-    {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"},
-    {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"},
+    {file = "PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086"},
+    {file = "PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf"},
+    {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237"},
+    {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b"},
+    {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed"},
+    {file = "PyYAML-6.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180"},
+    {file = "PyYAML-6.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68"},
+    {file = "PyYAML-6.0.2-cp310-cp310-win32.whl", hash = "sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99"},
+    {file = "PyYAML-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e"},
+    {file = "PyYAML-6.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774"},
+    {file = "PyYAML-6.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee"},
+    {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c"},
+    {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317"},
+    {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85"},
+    {file = "PyYAML-6.0.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4"},
+    {file = "PyYAML-6.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e"},
+    {file = "PyYAML-6.0.2-cp311-cp311-win32.whl", hash = "sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5"},
+    {file = "PyYAML-6.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44"},
+    {file = "PyYAML-6.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab"},
+    {file = "PyYAML-6.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725"},
+    {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5"},
+    {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425"},
+    {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476"},
+    {file = "PyYAML-6.0.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48"},
+    {file = "PyYAML-6.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b"},
+    {file = "PyYAML-6.0.2-cp312-cp312-win32.whl", hash = "sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4"},
+    {file = "PyYAML-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8"},
+    {file = "PyYAML-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba"},
+    {file = "PyYAML-6.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1"},
+    {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133"},
+    {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484"},
+    {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5"},
+    {file = "PyYAML-6.0.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc"},
+    {file = "PyYAML-6.0.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652"},
+    {file = "PyYAML-6.0.2-cp313-cp313-win32.whl", hash = "sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183"},
+    {file = "PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563"},
+    {file = "PyYAML-6.0.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:24471b829b3bf607e04e88d79542a9d48bb037c2267d7927a874e6c205ca7e9a"},
+    {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7fded462629cfa4b685c5416b949ebad6cec74af5e2d42905d41e257e0869f5"},
+    {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d84a1718ee396f54f3a086ea0a66d8e552b2ab2017ef8b420e92edbc841c352d"},
+    {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9056c1ecd25795207ad294bcf39f2db3d845767be0ea6e6a34d856f006006083"},
+    {file = "PyYAML-6.0.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:82d09873e40955485746739bcb8b4586983670466c23382c19cffecbf1fd8706"},
+    {file = "PyYAML-6.0.2-cp38-cp38-win32.whl", hash = "sha256:43fa96a3ca0d6b1812e01ced1044a003533c47f6ee8aca31724f78e93ccc089a"},
+    {file = "PyYAML-6.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:01179a4a8559ab5de078078f37e5c1a30d76bb88519906844fd7bdea1b7729ff"},
+    {file = "PyYAML-6.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:688ba32a1cffef67fd2e9398a2efebaea461578b0923624778664cc1c914db5d"},
+    {file = "PyYAML-6.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a8786accb172bd8afb8be14490a16625cbc387036876ab6ba70912730faf8e1f"},
+    {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8e03406cac8513435335dbab54c0d385e4a49e4945d2909a581c83647ca0290"},
+    {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f753120cb8181e736c57ef7636e83f31b9c0d1722c516f7e86cf15b7aa57ff12"},
+    {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b1fdb9dc17f5a7677423d508ab4f243a726dea51fa5e70992e59a7411c89d19"},
+    {file = "PyYAML-6.0.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0b69e4ce7a131fe56b7e4d770c67429700908fc0752af059838b1cfb41960e4e"},
+    {file = "PyYAML-6.0.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a9f8c2e67970f13b16084e04f134610fd1d374bf477b17ec1599185cf611d725"},
+    {file = "PyYAML-6.0.2-cp39-cp39-win32.whl", hash = "sha256:6395c297d42274772abc367baaa79683958044e5d3835486c16da75d2a694631"},
+    {file = "PyYAML-6.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:39693e1f8320ae4f43943590b49779ffb98acb81f788220ea932a6b6c51004d8"},
+    {file = "pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e"},
 ]
 
 [[package]]
 name = "requests"
-version = "2.31.0"
+version = "2.32.3"
 description = "Python HTTP for Humans."
 optional = false
-python-versions = ">=3.7"
+python-versions = ">=3.8"
 files = [
-    {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"},
-    {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"},
+    {file = "requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6"},
+    {file = "requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760"},
 ]
 
 [package.dependencies]
@@ -855,18 +976,17 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"]
 
 [[package]]
 name = "syrupy"
-version = "4.0.1"
+version = "4.7.1"
 description = "Pytest Snapshot Test Utility"
 optional = false
-python-versions = ">=3.8.1,<4"
+python-versions = ">=3.8.1"
 files = [
-    {file = "syrupy-4.0.1-py3-none-any.whl", hash = "sha256:53d3107cc5e18a5def189c721879cea2cdafdee34b879f602133ca08837d0e4b"},
-    {file = "syrupy-4.0.1.tar.gz", hash = "sha256:60e3e94782444e0f978cd3b207de32f6da3199b15a2db32eab02f83cebb63ae8"},
+    {file = "syrupy-4.7.1-py3-none-any.whl", hash = "sha256:be002267a512a4bedddfae2e026c93df1ea928ae10baadc09640516923376d41"},
+    {file = "syrupy-4.7.1.tar.gz", hash = "sha256:f9d4485f3f27d0e5df6ed299cac6fa32eb40a441915d988e82be5a4bdda335c8"},
 ]
 
 [package.dependencies]
-colored = ">=1.3.92,<2.0.0"
-pytest = ">=7.0.0,<8.0.0"
+pytest = ">=7.0.0,<9.0.0"
 
 [[package]]
 name = "text-generation"
@@ -897,13 +1017,13 @@ files = [
 
 [[package]]
 name = "tqdm"
-version = "4.66.1"
+version = "4.66.5"
 description = "Fast, Extensible Progress Meter"
 optional = false
 python-versions = ">=3.7"
 files = [
-    {file = "tqdm-4.66.1-py3-none-any.whl", hash = "sha256:d302b3c5b53d47bce91fea46679d9c3c6508cf6332229aa1e7d8653723793386"},
-    {file = "tqdm-4.66.1.tar.gz", hash = "sha256:d88e651f9db8d8551a62556d3cff9e3034274ca5d66e93197cf2490e2dcb69c7"},
+    {file = "tqdm-4.66.5-py3-none-any.whl", hash = "sha256:90279a3770753eafc9194a0364852159802111925aa30eb3f9d85b0e805ac7cd"},
+    {file = "tqdm-4.66.5.tar.gz", hash = "sha256:e1020aef2e5096702d8a025ac7d16b1577279c9d63f8375b63083e9a5f0fcbad"},
 ]
 
 [package.dependencies]
@@ -917,129 +1037,131 @@ telegram = ["requests"]
 
 [[package]]
 name = "typing-extensions"
-version = "4.7.1"
-description = "Backported and Experimental Type Hints for Python 3.7+"
+version = "4.12.2"
+description = "Backported and Experimental Type Hints for Python 3.8+"
 optional = false
-python-versions = ">=3.7"
+python-versions = ">=3.8"
 files = [
-    {file = "typing_extensions-4.7.1-py3-none-any.whl", hash = "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36"},
-    {file = "typing_extensions-4.7.1.tar.gz", hash = "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2"},
+    {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"},
+    {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"},
 ]
 
 [[package]]
 name = "urllib3"
-version = "2.0.4"
+version = "2.2.2"
 description = "HTTP library with thread-safe connection pooling, file post, and more."
 optional = false
-python-versions = ">=3.7"
+python-versions = ">=3.8"
 files = [
-    {file = "urllib3-2.0.4-py3-none-any.whl", hash = "sha256:de7df1803967d2c2a98e4b11bb7d6bd9210474c46e8a0401514e3a42a75ebde4"},
-    {file = "urllib3-2.0.4.tar.gz", hash = "sha256:8d22f86aae8ef5e410d4f539fde9ce6b2113a001bb4d189e0aed70642d602b11"},
+    {file = "urllib3-2.2.2-py3-none-any.whl", hash = "sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472"},
+    {file = "urllib3-2.2.2.tar.gz", hash = "sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168"},
 ]
 
 [package.extras]
 brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"]
-secure = ["certifi", "cryptography (>=1.9)", "idna (>=2.0.0)", "pyopenssl (>=17.1.0)", "urllib3-secure-extra"]
+h2 = ["h2 (>=4,<5)"]
 socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"]
 zstd = ["zstandard (>=0.18.0)"]
 
 [[package]]
-name = "websocket-client"
-version = "1.6.2"
-description = "WebSocket client for Python with low level API options"
+name = "yarl"
+version = "1.11.1"
+description = "Yet another URL library"
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "websocket-client-1.6.2.tar.gz", hash = "sha256:53e95c826bf800c4c465f50093a8c4ff091c7327023b10bfaff40cf1ef170eaa"},
-    {file = "websocket_client-1.6.2-py3-none-any.whl", hash = "sha256:ce54f419dfae71f4bdba69ebe65bf7f0a93fe71bc009ad3a010aacc3eebad537"},
-]
-
-[package.extras]
-docs = ["Sphinx (>=6.0)", "sphinx-rtd-theme (>=1.1.0)"]
-optional = ["python-socks", "wsaccel"]
-test = ["websockets"]
-
-[[package]]
-name = "yarl"
-version = "1.9.2"
-description = "Yet another URL library"
-optional = false
-python-versions = ">=3.7"
-files = [
-    {file = "yarl-1.9.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c2ad583743d16ddbdf6bb14b5cd76bf43b0d0006e918809d5d4ddf7bde8dd82"},
-    {file = "yarl-1.9.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:82aa6264b36c50acfb2424ad5ca537a2060ab6de158a5bd2a72a032cc75b9eb8"},
-    {file = "yarl-1.9.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c0c77533b5ed4bcc38e943178ccae29b9bcf48ffd1063f5821192f23a1bd27b9"},
-    {file = "yarl-1.9.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ee4afac41415d52d53a9833ebae7e32b344be72835bbb589018c9e938045a560"},
-    {file = "yarl-1.9.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9bf345c3a4f5ba7f766430f97f9cc1320786f19584acc7086491f45524a551ac"},
-    {file = "yarl-1.9.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2a96c19c52ff442a808c105901d0bdfd2e28575b3d5f82e2f5fd67e20dc5f4ea"},
-    {file = "yarl-1.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:891c0e3ec5ec881541f6c5113d8df0315ce5440e244a716b95f2525b7b9f3608"},
-    {file = "yarl-1.9.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c3a53ba34a636a256d767c086ceb111358876e1fb6b50dfc4d3f4951d40133d5"},
-    {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:566185e8ebc0898b11f8026447eacd02e46226716229cea8db37496c8cdd26e0"},
-    {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:2b0738fb871812722a0ac2154be1f049c6223b9f6f22eec352996b69775b36d4"},
-    {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:32f1d071b3f362c80f1a7d322bfd7b2d11e33d2adf395cc1dd4df36c9c243095"},
-    {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:e9fdc7ac0d42bc3ea78818557fab03af6181e076a2944f43c38684b4b6bed8e3"},
-    {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:56ff08ab5df8429901ebdc5d15941b59f6253393cb5da07b4170beefcf1b2528"},
-    {file = "yarl-1.9.2-cp310-cp310-win32.whl", hash = "sha256:8ea48e0a2f931064469bdabca50c2f578b565fc446f302a79ba6cc0ee7f384d3"},
-    {file = "yarl-1.9.2-cp310-cp310-win_amd64.whl", hash = "sha256:50f33040f3836e912ed16d212f6cc1efb3231a8a60526a407aeb66c1c1956dde"},
-    {file = "yarl-1.9.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:646d663eb2232d7909e6601f1a9107e66f9791f290a1b3dc7057818fe44fc2b6"},
-    {file = "yarl-1.9.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:aff634b15beff8902d1f918012fc2a42e0dbae6f469fce134c8a0dc51ca423bb"},
-    {file = "yarl-1.9.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a83503934c6273806aed765035716216cc9ab4e0364f7f066227e1aaea90b8d0"},
-    {file = "yarl-1.9.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b25322201585c69abc7b0e89e72790469f7dad90d26754717f3310bfe30331c2"},
-    {file = "yarl-1.9.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:22a94666751778629f1ec4280b08eb11815783c63f52092a5953faf73be24191"},
-    {file = "yarl-1.9.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8ec53a0ea2a80c5cd1ab397925f94bff59222aa3cf9c6da938ce05c9ec20428d"},
-    {file = "yarl-1.9.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:159d81f22d7a43e6eabc36d7194cb53f2f15f498dbbfa8edc8a3239350f59fe7"},
-    {file = "yarl-1.9.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:832b7e711027c114d79dffb92576acd1bd2decc467dec60e1cac96912602d0e6"},
-    {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:95d2ecefbcf4e744ea952d073c6922e72ee650ffc79028eb1e320e732898d7e8"},
-    {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:d4e2c6d555e77b37288eaf45b8f60f0737c9efa3452c6c44626a5455aeb250b9"},
-    {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:783185c75c12a017cc345015ea359cc801c3b29a2966c2655cd12b233bf5a2be"},
-    {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:b8cc1863402472f16c600e3e93d542b7e7542a540f95c30afd472e8e549fc3f7"},
-    {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:822b30a0f22e588b32d3120f6d41e4ed021806418b4c9f0bc3048b8c8cb3f92a"},
-    {file = "yarl-1.9.2-cp311-cp311-win32.whl", hash = "sha256:a60347f234c2212a9f0361955007fcf4033a75bf600a33c88a0a8e91af77c0e8"},
-    {file = "yarl-1.9.2-cp311-cp311-win_amd64.whl", hash = "sha256:be6b3fdec5c62f2a67cb3f8c6dbf56bbf3f61c0f046f84645cd1ca73532ea051"},
-    {file = "yarl-1.9.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:38a3928ae37558bc1b559f67410df446d1fbfa87318b124bf5032c31e3447b74"},
-    {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac9bb4c5ce3975aeac288cfcb5061ce60e0d14d92209e780c93954076c7c4367"},
-    {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3da8a678ca8b96c8606bbb8bfacd99a12ad5dd288bc6f7979baddd62f71c63ef"},
-    {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:13414591ff516e04fcdee8dc051c13fd3db13b673c7a4cb1350e6b2ad9639ad3"},
-    {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf74d08542c3a9ea97bb8f343d4fcbd4d8f91bba5ec9d5d7f792dbe727f88938"},
-    {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6e7221580dc1db478464cfeef9b03b95c5852cc22894e418562997df0d074ccc"},
-    {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:494053246b119b041960ddcd20fd76224149cfea8ed8777b687358727911dd33"},
-    {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:52a25809fcbecfc63ac9ba0c0fb586f90837f5425edfd1ec9f3372b119585e45"},
-    {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:e65610c5792870d45d7b68c677681376fcf9cc1c289f23e8e8b39c1485384185"},
-    {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:1b1bba902cba32cdec51fca038fd53f8beee88b77efc373968d1ed021024cc04"},
-    {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:662e6016409828ee910f5d9602a2729a8a57d74b163c89a837de3fea050c7582"},
-    {file = "yarl-1.9.2-cp37-cp37m-win32.whl", hash = "sha256:f364d3480bffd3aa566e886587eaca7c8c04d74f6e8933f3f2c996b7f09bee1b"},
-    {file = "yarl-1.9.2-cp37-cp37m-win_amd64.whl", hash = "sha256:6a5883464143ab3ae9ba68daae8e7c5c95b969462bbe42e2464d60e7e2698368"},
-    {file = "yarl-1.9.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5610f80cf43b6202e2c33ba3ec2ee0a2884f8f423c8f4f62906731d876ef4fac"},
-    {file = "yarl-1.9.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b9a4e67ad7b646cd6f0938c7ebfd60e481b7410f574c560e455e938d2da8e0f4"},
-    {file = "yarl-1.9.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:83fcc480d7549ccebe9415d96d9263e2d4226798c37ebd18c930fce43dfb9574"},
-    {file = "yarl-1.9.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5fcd436ea16fee7d4207c045b1e340020e58a2597301cfbcfdbe5abd2356c2fb"},
-    {file = "yarl-1.9.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:84e0b1599334b1e1478db01b756e55937d4614f8654311eb26012091be109d59"},
-    {file = "yarl-1.9.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3458a24e4ea3fd8930e934c129b676c27452e4ebda80fbe47b56d8c6c7a63a9e"},
-    {file = "yarl-1.9.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:838162460b3a08987546e881a2bfa573960bb559dfa739e7800ceeec92e64417"},
-    {file = "yarl-1.9.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f4e2d08f07a3d7d3e12549052eb5ad3eab1c349c53ac51c209a0e5991bbada78"},
-    {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:de119f56f3c5f0e2fb4dee508531a32b069a5f2c6e827b272d1e0ff5ac040333"},
-    {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:149ddea5abf329752ea5051b61bd6c1d979e13fbf122d3a1f9f0c8be6cb6f63c"},
-    {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:674ca19cbee4a82c9f54e0d1eee28116e63bc6fd1e96c43031d11cbab8b2afd5"},
-    {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:9b3152f2f5677b997ae6c804b73da05a39daa6a9e85a512e0e6823d81cdad7cc"},
-    {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:5415d5a4b080dc9612b1b63cba008db84e908b95848369aa1da3686ae27b6d2b"},
-    {file = "yarl-1.9.2-cp38-cp38-win32.whl", hash = "sha256:f7a3d8146575e08c29ed1cd287068e6d02f1c7bdff8970db96683b9591b86ee7"},
-    {file = "yarl-1.9.2-cp38-cp38-win_amd64.whl", hash = "sha256:63c48f6cef34e6319a74c727376e95626f84ea091f92c0250a98e53e62c77c72"},
-    {file = "yarl-1.9.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:75df5ef94c3fdc393c6b19d80e6ef1ecc9ae2f4263c09cacb178d871c02a5ba9"},
-    {file = "yarl-1.9.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c027a6e96ef77d401d8d5a5c8d6bc478e8042f1e448272e8d9752cb0aff8b5c8"},
-    {file = "yarl-1.9.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f3b078dbe227f79be488ffcfc7a9edb3409d018e0952cf13f15fd6512847f3f7"},
-    {file = "yarl-1.9.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:59723a029760079b7d991a401386390c4be5bfec1e7dd83e25a6a0881859e716"},
-    {file = "yarl-1.9.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b03917871bf859a81ccb180c9a2e6c1e04d2f6a51d953e6a5cdd70c93d4e5a2a"},
-    {file = "yarl-1.9.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c1012fa63eb6c032f3ce5d2171c267992ae0c00b9e164efe4d73db818465fac3"},
-    {file = "yarl-1.9.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a74dcbfe780e62f4b5a062714576f16c2f3493a0394e555ab141bf0d746bb955"},
-    {file = "yarl-1.9.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8c56986609b057b4839968ba901944af91b8e92f1725d1a2d77cbac6972b9ed1"},
-    {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:2c315df3293cd521033533d242d15eab26583360b58f7ee5d9565f15fee1bef4"},
-    {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:b7232f8dfbd225d57340e441d8caf8652a6acd06b389ea2d3222b8bc89cbfca6"},
-    {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:53338749febd28935d55b41bf0bcc79d634881195a39f6b2f767870b72514caf"},
-    {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:066c163aec9d3d073dc9ffe5dd3ad05069bcb03fcaab8d221290ba99f9f69ee3"},
-    {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8288d7cd28f8119b07dd49b7230d6b4562f9b61ee9a4ab02221060d21136be80"},
-    {file = "yarl-1.9.2-cp39-cp39-win32.whl", hash = "sha256:b124e2a6d223b65ba8768d5706d103280914d61f5cae3afbc50fc3dfcc016623"},
-    {file = "yarl-1.9.2-cp39-cp39-win_amd64.whl", hash = "sha256:61016e7d582bc46a5378ffdd02cd0314fb8ba52f40f9cf4d9a5e7dbef88dee18"},
-    {file = "yarl-1.9.2.tar.gz", hash = "sha256:04ab9d4b9f587c06d801c2abfe9317b77cdf996c65a90d5e84ecc45010823571"},
+    {file = "yarl-1.11.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:400cd42185f92de559d29eeb529e71d80dfbd2f45c36844914a4a34297ca6f00"},
+    {file = "yarl-1.11.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8258c86f47e080a258993eed877d579c71da7bda26af86ce6c2d2d072c11320d"},
+    {file = "yarl-1.11.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2164cd9725092761fed26f299e3f276bb4b537ca58e6ff6b252eae9631b5c96e"},
+    {file = "yarl-1.11.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08ea567c16f140af8ddc7cb58e27e9138a1386e3e6e53982abaa6f2377b38cc"},
+    {file = "yarl-1.11.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:768ecc550096b028754ea28bf90fde071c379c62c43afa574edc6f33ee5daaec"},
+    {file = "yarl-1.11.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2909fa3a7d249ef64eeb2faa04b7957e34fefb6ec9966506312349ed8a7e77bf"},
+    {file = "yarl-1.11.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:01a8697ec24f17c349c4f655763c4db70eebc56a5f82995e5e26e837c6eb0e49"},
+    {file = "yarl-1.11.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e286580b6511aac7c3268a78cdb861ec739d3e5a2a53b4809faef6b49778eaff"},
+    {file = "yarl-1.11.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4179522dc0305c3fc9782549175c8e8849252fefeb077c92a73889ccbcd508ad"},
+    {file = "yarl-1.11.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:27fcb271a41b746bd0e2a92182df507e1c204759f460ff784ca614e12dd85145"},
+    {file = "yarl-1.11.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:f61db3b7e870914dbd9434b560075e0366771eecbe6d2b5561f5bc7485f39efd"},
+    {file = "yarl-1.11.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:c92261eb2ad367629dc437536463dc934030c9e7caca861cc51990fe6c565f26"},
+    {file = "yarl-1.11.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d95b52fbef190ca87d8c42f49e314eace4fc52070f3dfa5f87a6594b0c1c6e46"},
+    {file = "yarl-1.11.1-cp310-cp310-win32.whl", hash = "sha256:489fa8bde4f1244ad6c5f6d11bb33e09cf0d1d0367edb197619c3e3fc06f3d91"},
+    {file = "yarl-1.11.1-cp310-cp310-win_amd64.whl", hash = "sha256:476e20c433b356e16e9a141449f25161e6b69984fb4cdbd7cd4bd54c17844998"},
+    {file = "yarl-1.11.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:946eedc12895873891aaceb39bceb484b4977f70373e0122da483f6c38faaa68"},
+    {file = "yarl-1.11.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:21a7c12321436b066c11ec19c7e3cb9aec18884fe0d5b25d03d756a9e654edfe"},
+    {file = "yarl-1.11.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c35f493b867912f6fda721a59cc7c4766d382040bdf1ddaeeaa7fa4d072f4675"},
+    {file = "yarl-1.11.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25861303e0be76b60fddc1250ec5986c42f0a5c0c50ff57cc30b1be199c00e63"},
+    {file = "yarl-1.11.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e4b53f73077e839b3f89c992223f15b1d2ab314bdbdf502afdc7bb18e95eae27"},
+    {file = "yarl-1.11.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:327c724b01b8641a1bf1ab3b232fb638706e50f76c0b5bf16051ab65c868fac5"},
+    {file = "yarl-1.11.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4307d9a3417eea87715c9736d050c83e8c1904e9b7aada6ce61b46361b733d92"},
+    {file = "yarl-1.11.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:48a28bed68ab8fb7e380775f0029a079f08a17799cb3387a65d14ace16c12e2b"},
+    {file = "yarl-1.11.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:067b961853c8e62725ff2893226fef3d0da060656a9827f3f520fb1d19b2b68a"},
+    {file = "yarl-1.11.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:8215f6f21394d1f46e222abeb06316e77ef328d628f593502d8fc2a9117bde83"},
+    {file = "yarl-1.11.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:498442e3af2a860a663baa14fbf23fb04b0dd758039c0e7c8f91cb9279799bff"},
+    {file = "yarl-1.11.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:69721b8effdb588cb055cc22f7c5105ca6fdaa5aeb3ea09021d517882c4a904c"},
+    {file = "yarl-1.11.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:1e969fa4c1e0b1a391f3fcbcb9ec31e84440253325b534519be0d28f4b6b533e"},
+    {file = "yarl-1.11.1-cp311-cp311-win32.whl", hash = "sha256:7d51324a04fc4b0e097ff8a153e9276c2593106a811704025bbc1d6916f45ca6"},
+    {file = "yarl-1.11.1-cp311-cp311-win_amd64.whl", hash = "sha256:15061ce6584ece023457fb8b7a7a69ec40bf7114d781a8c4f5dcd68e28b5c53b"},
+    {file = "yarl-1.11.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:a4264515f9117be204935cd230fb2a052dd3792789cc94c101c535d349b3dab0"},
+    {file = "yarl-1.11.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:f41fa79114a1d2eddb5eea7b912d6160508f57440bd302ce96eaa384914cd265"},
+    {file = "yarl-1.11.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:02da8759b47d964f9173c8675710720b468aa1c1693be0c9c64abb9d8d9a4867"},
+    {file = "yarl-1.11.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9361628f28f48dcf8b2f528420d4d68102f593f9c2e592bfc842f5fb337e44fd"},
+    {file = "yarl-1.11.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b91044952da03b6f95fdba398d7993dd983b64d3c31c358a4c89e3c19b6f7aef"},
+    {file = "yarl-1.11.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:74db2ef03b442276d25951749a803ddb6e270d02dda1d1c556f6ae595a0d76a8"},
+    {file = "yarl-1.11.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e975a2211952a8a083d1b9d9ba26472981ae338e720b419eb50535de3c02870"},
+    {file = "yarl-1.11.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8aef97ba1dd2138112890ef848e17d8526fe80b21f743b4ee65947ea184f07a2"},
+    {file = "yarl-1.11.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a7915ea49b0c113641dc4d9338efa9bd66b6a9a485ffe75b9907e8573ca94b84"},
+    {file = "yarl-1.11.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:504cf0d4c5e4579a51261d6091267f9fd997ef58558c4ffa7a3e1460bd2336fa"},
+    {file = "yarl-1.11.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:3de5292f9f0ee285e6bd168b2a77b2a00d74cbcfa420ed078456d3023d2f6dff"},
+    {file = "yarl-1.11.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:a34e1e30f1774fa35d37202bbeae62423e9a79d78d0874e5556a593479fdf239"},
+    {file = "yarl-1.11.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:66b63c504d2ca43bf7221a1f72fbe981ff56ecb39004c70a94485d13e37ebf45"},
+    {file = "yarl-1.11.1-cp312-cp312-win32.whl", hash = "sha256:a28b70c9e2213de425d9cba5ab2e7f7a1c8ca23a99c4b5159bf77b9c31251447"},
+    {file = "yarl-1.11.1-cp312-cp312-win_amd64.whl", hash = "sha256:17b5a386d0d36fb828e2fb3ef08c8829c1ebf977eef88e5367d1c8c94b454639"},
+    {file = "yarl-1.11.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:1fa2e7a406fbd45b61b4433e3aa254a2c3e14c4b3186f6e952d08a730807fa0c"},
+    {file = "yarl-1.11.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:750f656832d7d3cb0c76be137ee79405cc17e792f31e0a01eee390e383b2936e"},
+    {file = "yarl-1.11.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0b8486f322d8f6a38539136a22c55f94d269addb24db5cb6f61adc61eabc9d93"},
+    {file = "yarl-1.11.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3fce4da3703ee6048ad4138fe74619c50874afe98b1ad87b2698ef95bf92c96d"},
+    {file = "yarl-1.11.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8ed653638ef669e0efc6fe2acb792275cb419bf9cb5c5049399f3556995f23c7"},
+    {file = "yarl-1.11.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:18ac56c9dd70941ecad42b5a906820824ca72ff84ad6fa18db33c2537ae2e089"},
+    {file = "yarl-1.11.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:688654f8507464745ab563b041d1fb7dab5d9912ca6b06e61d1c4708366832f5"},
+    {file = "yarl-1.11.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4973eac1e2ff63cf187073cd4e1f1148dcd119314ab79b88e1b3fad74a18c9d5"},
+    {file = "yarl-1.11.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:964a428132227edff96d6f3cf261573cb0f1a60c9a764ce28cda9525f18f7786"},
+    {file = "yarl-1.11.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:6d23754b9939cbab02c63434776df1170e43b09c6a517585c7ce2b3d449b7318"},
+    {file = "yarl-1.11.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:c2dc4250fe94d8cd864d66018f8344d4af50e3758e9d725e94fecfa27588ff82"},
+    {file = "yarl-1.11.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:09696438cb43ea6f9492ef237761b043f9179f455f405279e609f2bc9100212a"},
+    {file = "yarl-1.11.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:999bfee0a5b7385a0af5ffb606393509cfde70ecca4f01c36985be6d33e336da"},
+    {file = "yarl-1.11.1-cp313-cp313-win32.whl", hash = "sha256:ce928c9c6409c79e10f39604a7e214b3cb69552952fbda8d836c052832e6a979"},
+    {file = "yarl-1.11.1-cp313-cp313-win_amd64.whl", hash = "sha256:501c503eed2bb306638ccb60c174f856cc3246c861829ff40eaa80e2f0330367"},
+    {file = "yarl-1.11.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:dae7bd0daeb33aa3e79e72877d3d51052e8b19c9025ecf0374f542ea8ec120e4"},
+    {file = "yarl-1.11.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:3ff6b1617aa39279fe18a76c8d165469c48b159931d9b48239065767ee455b2b"},
+    {file = "yarl-1.11.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3257978c870728a52dcce8c2902bf01f6c53b65094b457bf87b2644ee6238ddc"},
+    {file = "yarl-1.11.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0f351fa31234699d6084ff98283cb1e852270fe9e250a3b3bf7804eb493bd937"},
+    {file = "yarl-1.11.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8aef1b64da41d18026632d99a06b3fefe1d08e85dd81d849fa7c96301ed22f1b"},
+    {file = "yarl-1.11.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7175a87ab8f7fbde37160a15e58e138ba3b2b0e05492d7351314a250d61b1591"},
+    {file = "yarl-1.11.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba444bdd4caa2a94456ef67a2f383710928820dd0117aae6650a4d17029fa25e"},
+    {file = "yarl-1.11.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0ea9682124fc062e3d931c6911934a678cb28453f957ddccf51f568c2f2b5e05"},
+    {file = "yarl-1.11.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:8418c053aeb236b20b0ab8fa6bacfc2feaaf7d4683dd96528610989c99723d5f"},
+    {file = "yarl-1.11.1-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:61a5f2c14d0a1adfdd82258f756b23a550c13ba4c86c84106be4c111a3a4e413"},
+    {file = "yarl-1.11.1-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:f3a6d90cab0bdf07df8f176eae3a07127daafcf7457b997b2bf46776da2c7eb7"},
+    {file = "yarl-1.11.1-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:077da604852be488c9a05a524068cdae1e972b7dc02438161c32420fb4ec5e14"},
+    {file = "yarl-1.11.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:15439f3c5c72686b6c3ff235279630d08936ace67d0fe5c8d5bbc3ef06f5a420"},
+    {file = "yarl-1.11.1-cp38-cp38-win32.whl", hash = "sha256:238a21849dd7554cb4d25a14ffbfa0ef380bb7ba201f45b144a14454a72ffa5a"},
+    {file = "yarl-1.11.1-cp38-cp38-win_amd64.whl", hash = "sha256:67459cf8cf31da0e2cbdb4b040507e535d25cfbb1604ca76396a3a66b8ba37a6"},
+    {file = "yarl-1.11.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:884eab2ce97cbaf89f264372eae58388862c33c4f551c15680dd80f53c89a269"},
+    {file = "yarl-1.11.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8a336eaa7ee7e87cdece3cedb395c9657d227bfceb6781295cf56abcd3386a26"},
+    {file = "yarl-1.11.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:87f020d010ba80a247c4abc335fc13421037800ca20b42af5ae40e5fd75e7909"},
+    {file = "yarl-1.11.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:637c7ddb585a62d4469f843dac221f23eec3cbad31693b23abbc2c366ad41ff4"},
+    {file = "yarl-1.11.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:48dfd117ab93f0129084577a07287376cc69c08138694396f305636e229caa1a"},
+    {file = "yarl-1.11.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:75e0ae31fb5ccab6eda09ba1494e87eb226dcbd2372dae96b87800e1dcc98804"},
+    {file = "yarl-1.11.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f46f81501160c28d0c0b7333b4f7be8983dbbc161983b6fb814024d1b4952f79"},
+    {file = "yarl-1.11.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:04293941646647b3bfb1719d1d11ff1028e9c30199509a844da3c0f5919dc520"},
+    {file = "yarl-1.11.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:250e888fa62d73e721f3041e3a9abf427788a1934b426b45e1b92f62c1f68366"},
+    {file = "yarl-1.11.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:e8f63904df26d1a66aabc141bfd258bf738b9bc7bc6bdef22713b4f5ef789a4c"},
+    {file = "yarl-1.11.1-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:aac44097d838dda26526cffb63bdd8737a2dbdf5f2c68efb72ad83aec6673c7e"},
+    {file = "yarl-1.11.1-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:267b24f891e74eccbdff42241c5fb4f974de2d6271dcc7d7e0c9ae1079a560d9"},
+    {file = "yarl-1.11.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:6907daa4b9d7a688063ed098c472f96e8181733c525e03e866fb5db480a424df"},
+    {file = "yarl-1.11.1-cp39-cp39-win32.whl", hash = "sha256:14438dfc5015661f75f85bc5adad0743678eefee266ff0c9a8e32969d5d69f74"},
+    {file = "yarl-1.11.1-cp39-cp39-win_amd64.whl", hash = "sha256:94d0caaa912bfcdc702a4204cd5e2bb01eb917fc4f5ea2315aa23962549561b0"},
+    {file = "yarl-1.11.1-py3-none-any.whl", hash = "sha256:72bf26f66456baa0584eff63e44545c9f0eaed9b73cb6601b647c91f14c11f38"},
+    {file = "yarl-1.11.1.tar.gz", hash = "sha256:1bb2d9e212fb7449b8fb73bc461b51eaa17cc8430b4a87d87be7b25052d92f53"},
 ]
 
 [package.dependencies]
@@ -1048,5 +1170,5 @@ multidict = ">=4.0"
 
 [metadata]
 lock-version = "2.0"
-python-versions = ">=3.9,<3.13"
-content-hash = "421fbce065cb1499c666599cf0fd83a5ce8fb3bed09e83c16c3a3d6953b34026"
+python-versions = ">=3.10,<3.13"
+content-hash = "310c0a2349bc0a0713b50f8482b4df4d07c48e0ce2c1bd91c30872f52de3fe1c"
diff --git a/integration-tests/pyproject.toml b/integration-tests/pyproject.toml
index 88e9761a..afd57ea7 100644
--- a/integration-tests/pyproject.toml
+++ b/integration-tests/pyproject.toml
@@ -6,9 +6,10 @@ authors = ["Nicolas Patry <nicolas@huggingface.co>"]
 
 [tool.poetry.dependencies]
 pydantic = "> 2, < 3"
-python = ">=3.9,<3.13"
-syrupy = "4.0.1"
+python = ">=3.10,<3.13"
+syrupy = "^4.7.1"
 text-generation = "^0.6.0"
 pytest = "^7.4.0"
 pytest-asyncio = "^0.21.1"
-docker = "^6.1.3"
+docker = "^7"
+numpy = "^1.20"
diff --git a/integration-tests/requirements.txt b/integration-tests/requirements.txt
index 3c2ce11b..8bf6ba07 100644
--- a/integration-tests/requirements.txt
+++ b/integration-tests/requirements.txt
@@ -1,35 +1,35 @@
-aiohttp==3.8.5 ; python_version >= "3.9" and python_version < "3.13"
-aiosignal==1.3.1 ; python_version >= "3.9" and python_version < "3.13"
-annotated-types==0.6.0 ; python_version >= "3.9" and python_version < "3.13"
-async-timeout==4.0.3 ; python_version >= "3.9" and python_version < "3.13"
-attrs==23.1.0 ; python_version >= "3.9" and python_version < "3.13"
-certifi==2023.7.22 ; python_version >= "3.9" and python_version < "3.13"
-charset-normalizer==3.2.0 ; python_version >= "3.9" and python_version < "3.13"
-colorama==0.4.6 ; python_version >= "3.9" and python_version < "3.13" and (sys_platform == "win32" or platform_system == "Windows")
-colored==1.4.4 ; python_version >= "3.9" and python_version < "3.13"
-docker==6.1.3 ; python_version >= "3.9" and python_version < "3.13"
-exceptiongroup==1.1.3 ; python_version >= "3.9" and python_version < "3.11"
-filelock==3.12.3 ; python_version >= "3.9" and python_version < "3.13"
-frozenlist==1.4.0 ; python_version >= "3.9" and python_version < "3.13"
-fsspec==2023.6.0 ; python_version >= "3.9" and python_version < "3.13"
-huggingface-hub==0.16.4 ; python_version >= "3.9" and python_version < "3.13"
-idna==3.4 ; python_version >= "3.9" and python_version < "3.13"
-iniconfig==2.0.0 ; python_version >= "3.9" and python_version < "3.13"
-multidict==6.0.4 ; python_version >= "3.9" and python_version < "3.13"
-packaging==23.1 ; python_version >= "3.9" and python_version < "3.13"
-pluggy==1.3.0 ; python_version >= "3.9" and python_version < "3.13"
-pydantic-core==2.16.3 ; python_version >= "3.9" and python_version < "3.13"
-pydantic==2.6.4 ; python_version >= "3.9" and python_version < "3.13"
-pytest-asyncio==0.21.1 ; python_version >= "3.9" and python_version < "3.13"
-pytest==7.4.0 ; python_version >= "3.9" and python_version < "3.13"
-pywin32==306 ; python_version >= "3.9" and python_version < "3.13" and sys_platform == "win32"
-pyyaml==6.0.1 ; python_version >= "3.9" and python_version < "3.13"
-requests==2.31.0 ; python_version >= "3.9" and python_version < "3.13"
-syrupy==4.0.1 ; python_version >= "3.9" and python_version < "3.13"
-text-generation==0.6.1 ; python_version >= "3.9" and python_version < "3.13"
-tomli==2.0.1 ; python_version >= "3.9" and python_version < "3.11"
-tqdm==4.66.1 ; python_version >= "3.9" and python_version < "3.13"
-typing-extensions==4.7.1 ; python_version >= "3.9" and python_version < "3.13"
-urllib3==2.0.4 ; python_version >= "3.9" and python_version < "3.13"
-websocket-client==1.6.2 ; python_version >= "3.9" and python_version < "3.13"
-yarl==1.9.2 ; python_version >= "3.9" and python_version < "3.13"
+aiohappyeyeballs==2.4.0 ; python_version >= "3.10" and python_version < "3.13"
+aiohttp==3.10.5 ; python_version >= "3.10" and python_version < "3.13"
+aiosignal==1.3.1 ; python_version >= "3.10" and python_version < "3.13"
+annotated-types==0.7.0 ; python_version >= "3.10" and python_version < "3.13"
+async-timeout==4.0.3 ; python_version >= "3.10" and python_version < "3.11"
+attrs==24.2.0 ; python_version >= "3.10" and python_version < "3.13"
+certifi==2024.8.30 ; python_version >= "3.10" and python_version < "3.13"
+charset-normalizer==3.3.2 ; python_version >= "3.10" and python_version < "3.13"
+colorama==0.4.6 ; python_version >= "3.10" and python_version < "3.13" and (sys_platform == "win32" or platform_system == "Windows")
+docker==7.1.0 ; python_version >= "3.10" and python_version < "3.13"
+exceptiongroup==1.2.2 ; python_version >= "3.10" and python_version < "3.11"
+filelock==3.16.0 ; python_version >= "3.10" and python_version < "3.13"
+frozenlist==1.4.1 ; python_version >= "3.10" and python_version < "3.13"
+fsspec==2024.9.0 ; python_version >= "3.10" and python_version < "3.13"
+huggingface-hub==0.24.6 ; python_version >= "3.10" and python_version < "3.13"
+idna==3.8 ; python_version >= "3.10" and python_version < "3.13"
+iniconfig==2.0.0 ; python_version >= "3.10" and python_version < "3.13"
+multidict==6.1.0 ; python_version >= "3.10" and python_version < "3.13"
+numpy==1.26.4 ; python_version >= "3.10" and python_version < "3.13"
+packaging==24.1 ; python_version >= "3.10" and python_version < "3.13"
+pluggy==1.5.0 ; python_version >= "3.10" and python_version < "3.13"
+pydantic-core==2.23.3 ; python_version >= "3.10" and python_version < "3.13"
+pydantic==2.9.1 ; python_version >= "3.10" and python_version < "3.13"
+pytest-asyncio==0.21.2 ; python_version >= "3.10" and python_version < "3.13"
+pytest==7.4.4 ; python_version >= "3.10" and python_version < "3.13"
+pywin32==306 ; python_version >= "3.10" and python_version < "3.13" and sys_platform == "win32"
+pyyaml==6.0.2 ; python_version >= "3.10" and python_version < "3.13"
+requests==2.32.3 ; python_version >= "3.10" and python_version < "3.13"
+syrupy==4.7.1 ; python_version >= "3.10" and python_version < "3.13"
+text-generation==0.6.1 ; python_version >= "3.10" and python_version < "3.13"
+tomli==2.0.1 ; python_version >= "3.10" and python_version < "3.11"
+tqdm==4.66.5 ; python_version >= "3.10" and python_version < "3.13"
+typing-extensions==4.12.2 ; python_version >= "3.10" and python_version < "3.13"
+urllib3==2.2.2 ; python_version >= "3.10" and python_version < "3.13"
+yarl==1.11.1 ; python_version >= "3.10" and python_version < "3.13"
diff --git a/launcher/Cargo.toml b/launcher/Cargo.toml
index eb219423..fdc3c02c 100644
--- a/launcher/Cargo.toml
+++ b/launcher/Cargo.toml
@@ -12,11 +12,13 @@ ctrlc = { version = "3.4.1", features = ["termination"] }
 hf-hub = "0.3.2"
 nix = { version = "0.28.0", features = ["signal"] }
 once_cell = "1.19.0"
+pyo3 = { workspace = true }
 serde = { version = "1.0.188", features = ["derive"] }
 serde_json = "1.0.107"
 thiserror = "1.0.59"
 tracing = "0.1.37"
 tracing-subscriber = { version = "0.3.17", features = ["json", "env-filter"] }
+regex = "1.11.0"
 
 [dev-dependencies]
 float_eq = "1.0.1"
diff --git a/launcher/src/gpu.rs b/launcher/src/gpu.rs
new file mode 100644
index 00000000..b565220e
--- /dev/null
+++ b/launcher/src/gpu.rs
@@ -0,0 +1,21 @@
+pub fn get_cuda_capability() -> Option<(usize, usize)> {
+    use pyo3::prelude::*;
+
+    let py_get_capability = |py: Python| -> PyResult<(isize, isize)> {
+        let torch = py.import_bound("torch.cuda")?;
+        let get_device_capability = torch.getattr("get_device_capability")?;
+        get_device_capability.call0()?.extract()
+    };
+
+    match pyo3::Python::with_gil(py_get_capability) {
+        Ok((major, minor)) if major < 0 || minor < 0 => {
+            tracing::warn!("Ignoring negative GPU compute capabilities: {major}.{minor}");
+            None
+        }
+        Ok((major, minor)) => Some((major as usize, minor as usize)),
+        Err(err) => {
+            tracing::warn!("Cannot determine GPU compute capability: {}", err);
+            None
+        }
+    }
+}
diff --git a/launcher/src/main.rs b/launcher/src/main.rs
index effd0b46..b3f07ef7 100644
--- a/launcher/src/main.rs
+++ b/launcher/src/main.rs
@@ -1,13 +1,15 @@
-/// Copyright (C) 2024 Habana Labs, Ltd. an Intel Company.
-
 use clap::{Parser, ValueEnum};
-use hf_hub::{api::sync::Api, Repo, RepoType};
+use hf_hub::{
+    api::sync::{Api, ApiBuilder},
+    Repo, RepoType,
+};
 use nix::sys::signal::{self, Signal};
 use nix::unistd::Pid;
+use regex::Regex;
 use serde::Deserialize;
 use std::env;
 use std::ffi::OsString;
-use std::io::{BufRead, BufReader, Lines};
+use std::io::{BufRead, BufReader};
 use std::os::unix::process::{CommandExt, ExitStatusExt};
 use std::path::Path;
 use std::process::{Child, Command, ExitStatus, Stdio};
@@ -17,22 +19,149 @@ use std::sync::{mpsc, Arc};
 use std::thread;
 use std::thread::sleep;
 use std::time::{Duration, Instant};
-use std::{fs, io};
+use std::{
+    fs, io,
+    io::{Read, Write},
+};
 use thiserror::Error;
 use tracing_subscriber::{filter::LevelFilter, EnvFilter};
 
 mod env_runtime;
+mod gpu;
+
+fn get_config(
+    model_id: &str,
+    revision: &Option<String>,
+) -> Result<Config, Box<dyn std::error::Error>> {
+    let mut path = std::path::Path::new(model_id).to_path_buf();
+    let model_id = model_id.to_string();
+    let filename = if !path.exists() {
+        // Assume it's a hub id
+
+        let api = if let Ok(token) = std::env::var("HF_TOKEN") {
+            // env variable has precedence over on file token.
+            ApiBuilder::new().with_token(Some(token)).build()?
+        } else {
+            Api::new()?
+        };
+        let repo = if let Some(ref revision) = revision {
+            api.repo(Repo::with_revision(
+                model_id,
+                RepoType::Model,
+                revision.to_string(),
+            ))
+        } else {
+            api.model(model_id)
+        };
+        repo.get("config.json")?
+    } else {
+        path.push("config.json");
+        path
+    };
+
+    let content = std::fs::read_to_string(filename)?;
+    let config: RawConfig = serde_json::from_str(&content)?;
+
+    let config: Config = config.into();
+    Ok(config)
+}
+
+fn resolve_attention(config: &Option<Config>, lora_adapters: &Option<String>) -> (String, String) {
+    let compute_capability = gpu::get_cuda_capability();
+    let mut prefix_caching: Option<String> = std::env::var("USE_PREFIX_CACHING").ok();
+    let mut attention: Option<String> = std::env::var("ATTENTION").ok();
+    if let Some(config) = config {
+        if prefix_caching.is_none() {
+            if config.vision_config.is_some() {
+                tracing::info!("Disabling prefix caching because of VLM model");
+                prefix_caching = Some("0".to_string());
+            } else if config.is_encoder_decoder {
+                tracing::info!("Disabling prefix caching because of seq2seq model");
+                prefix_caching = Some("0".to_string());
+            }
+        }
+
+        let fallback_attention = if matches!(compute_capability, Some((major, _)) if major < 8) {
+            "paged"
+        } else {
+            "flashdecoding"
+        };
+
+        match config.head_dim {
+            Some(h) if h == 64 || h == 128 || h == 256 => {
+                if lora_adapters.is_some() && prefix_caching.is_none() {
+                    tracing::info!("Disabling prefix caching because of lora adapters");
+                    prefix_caching = Some("0".to_string());
+                }
+                match config.model_type.as_deref() {
+                    Some("gemma2") | Some("falcon") | Some("deepseek_v2") => {
+                        // Required because gemma2 needs bfloat16 which is not supported by
+                        // flashinfer ?
+                        if attention.is_none() {
+                            tracing::info!(
+                                "Forcing attention to '{fallback_attention}' because model {} requires it",
+                                config.model_type.as_ref().unwrap()
+                            );
+                            attention = Some(fallback_attention.to_string());
+                        }
+                        if fallback_attention == "paged" && prefix_caching.is_none() {
+                            tracing::info!("Disabling prefix caching because it is not supported with 'paged' attention");
+                            prefix_caching = Some("0".to_string());
+                        }
+                    }
+                    Some("t5") => {}
+                    _ => {}
+                }
+            }
+            _ => {
+                if attention.is_none() {
+                    tracing::info!("Forcing attention to '{fallback_attention}' because head dim is not supported by flashinfer, also disabling prefix caching");
+                    attention = Some(fallback_attention.to_string());
+                }
+                if prefix_caching.is_none() {
+                    prefix_caching = Some("0".to_string());
+                }
+            }
+        }
+    }
+
+    let attention = attention.unwrap_or("flashinfer".to_string());
+    let prefix_caching = prefix_caching.unwrap_or("true".to_string());
+
+    (prefix_caching, attention)
+}
 
 #[derive(Deserialize)]
 struct RawConfig {
     max_position_embeddings: Option<usize>,
     n_positions: Option<usize>,
+    model_type: Option<String>,
     max_seq_len: Option<usize>,
+    quantization_config: Option<QuantizationConfig>,
+    n_embd: Option<usize>,
+    hidden_size: Option<usize>,
+    num_attention_heads: Option<usize>,
+    head_dim: Option<usize>,
+    vision_config: Option<VisionConfig>,
+    is_encoder_decoder: Option<bool>,
 }
 
+#[derive(Deserialize)]
+struct QuantizationConfig {
+    quant_method: Option<Quantization>,
+}
+
+#[derive(Deserialize)]
+struct VisionConfig {}
+
 #[derive(Deserialize)]
 struct Config {
     max_position_embeddings: Option<usize>,
+    quantize: Option<Quantization>,
+    head_dim: Option<usize>,
+    model_type: Option<String>,
+    vision_config: Option<VisionConfig>,
+    is_encoder_decoder: bool,
 }
 
 impl From<RawConfig> for Config {
@@ -41,13 +170,39 @@ impl From<RawConfig> for Config {
             .max_position_embeddings
             .or(other.max_seq_len)
             .or(other.n_positions);
+        let quantize = other.quantization_config.and_then(|q| q.quant_method);
+        let head_dim = other.head_dim.or_else(|| {
+            match (other.hidden_size, other.n_embd, other.num_attention_heads) {
+                (Some(hidden_size), _, Some(num_attention_heads))
+                    if hidden_size % num_attention_heads == 0 =>
+                {
+                    Some(hidden_size / num_attention_heads)
+                }
+                // Legacy
+                (_, Some(hidden_size), Some(num_attention_heads))
+                    if hidden_size % num_attention_heads == 0 =>
+                {
+                    Some(hidden_size / num_attention_heads)
+                }
+                _ => None,
+            }
+        });
+        let model_type = other.model_type;
+        let vision_config = other.vision_config;
+        let is_encoder_decoder = other.is_encoder_decoder.unwrap_or(false);
         Config {
             max_position_embeddings,
+            quantize,
+            head_dim,
+            model_type,
+            vision_config,
+            is_encoder_decoder,
         }
     }
 }
 
-#[derive(Clone, Copy, Debug, ValueEnum)]
+#[derive(Clone, Copy, Debug, ValueEnum, Deserialize)]
+#[serde(rename_all = "kebab-case")]
 enum Quantization {
     /// 4 bit quantization. Requires a specific AWQ quantized model:
     ///   <https://hf.co/models?search=awq>.
@@ -57,24 +212,30 @@ enum Quantization {
     /// Should be a drop-in replacement to bitsandbytes with much better performance.
     /// Kernels are from <https://github.com/NetEase-FuXi/EETQ.git>
     Eetq,
+    /// Variable bit quantization. Requires a specific EXL2 quantized model:
+    /// <https://hf.co/models?search=exl2>. Requires exllama2 kernels and does
+    /// not support tensor parallelism (num_shard > 1).
+    Exl2,
     /// 4 bit quantization. Requires a specific GTPQ quantized model: <https://hf.co/models?search=gptq>.
     /// text-generation-inference will use exllama (faster) kernels wherever possible, and use
     /// triton kernel (wider support) when it's not.
     /// AWQ has faster kernels.
     Gptq,
+    /// 4 bit quantization. Requires a specific Marlin quantized model: <https://hf.co/models?search=marlin>.
+    Marlin,
     /// Bitsandbytes 8bit. Can be applied on any model, will cut the memory requirement in half,
     /// but it is known that the model will be much slower to run than the native f16.
-    #[deprecated(
-        since = "1.1.0",
-        note = "Use `eetq` instead, which provides better latencies overall and is drop-in in most cases"
-    )]
+    // #[deprecated(
+    //     since = "1.1.0",
+    //     note = "Use `eetq` instead, which provides better latencies overall and is drop-in in most cases"
+    // )]
     Bitsandbytes,
     /// Bitsandbytes 4bit. Can be applied on any model, will cut the memory requirement by 4x,
     /// but it is known that the model will be much slower to run than the native f16.
-    BitsandbytesNF4,
+    BitsandbytesNf4,
     /// Bitsandbytes 4bit. nf4 should be preferred in most cases but maybe this one has better
     /// perplexity performance for you model
-    BitsandbytesFP4,
+    BitsandbytesFp4,
     /// [FP8](https://developer.nvidia.com/blog/nvidia-arm-and-intel-publish-fp8-specification-for-standardization-as-an-interchange-format-for-ai/) (e4m3) works on H100 and above
     /// This dtype has native ops should be the fastest if available.
     /// This is currently not the fastest because of local unpacking + padding to satisfy matrix
@@ -91,15 +252,21 @@ impl std::fmt::Display for Quantization {
             Quantization::Bitsandbytes => {
                 write!(f, "bitsandbytes")
             }
-            Quantization::BitsandbytesNF4 => {
+            Quantization::BitsandbytesNf4 => {
                 write!(f, "bitsandbytes-nf4")
             }
-            Quantization::BitsandbytesFP4 => {
+            Quantization::BitsandbytesFp4 => {
                 write!(f, "bitsandbytes-fp4")
             }
+            Quantization::Exl2 => {
+                write!(f, "exl2")
+            }
             Quantization::Gptq => {
                 write!(f, "gptq")
             }
+            Quantization::Marlin => {
+                write!(f, "marlin")
+            }
             Quantization::Awq => {
                 write!(f, "awq")
             }
@@ -154,6 +321,33 @@ impl std::fmt::Display for RopeScaling {
     }
 }
 
+#[derive(Clone, Copy, Debug, ValueEnum)]
+pub enum UsageStatsLevel {
+    /// Default option, usage statistics are collected anonymously
+    On,
+    /// Disables all collection of usage statistics
+    Off,
+    /// Doesn't send the error stack trace or error type, but allows sending a crash event
+    NoStack,
+}
+
+impl std::fmt::Display for UsageStatsLevel {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        // To keep in track with `server`.
+        match self {
+            UsageStatsLevel::On => {
+                write!(f, "on")
+            }
+            UsageStatsLevel::Off => {
+                write!(f, "off")
+            }
+            UsageStatsLevel::NoStack => {
+                write!(f, "no-stack")
+            }
+        }
+    }
+}
+
 /// App Configuration
 #[derive(Parser, Debug)]
 #[clap(author, version, about, long_about = None)]
@@ -189,7 +383,11 @@ struct Args {
     #[clap(long, env)]
     num_shard: Option<usize>,
 
-    /// Whether you want the model to be quantized.
+    /// Quantization method to use for the model. It is not necessary to specify this option
+    /// for pre-quantized models, since the quantization method is read from the model
+    /// configuration.
+    ///
+    /// Marlin kernels will be used automatically for GPTQ/AWQ models.
     #[clap(long, env, value_enum)]
     quantize: Option<Quantization>,
 
@@ -231,7 +429,7 @@ struct Args {
     max_stop_sequences: usize,
 
     /// This is the maximum allowed value for clients to set `top_n_tokens`.
-    /// `top_n_tokens is used to return information about the the `n` most likely
+    /// `top_n_tokens` is used to return information about the the `n` most likely
     /// tokens at each generation step, instead of just the sampled token. This
     /// information can be used for downstream tasks like for classification or
     /// ranking.
@@ -403,8 +601,15 @@ struct Args {
     #[clap(long, env)]
     otlp_endpoint: Option<String>,
 
+    #[clap(default_value = "text-generation-inference.router", long, env)]
+    otlp_service_name: String,
+
     #[clap(long, env)]
     cors_allow_origin: Vec<String>,
+
+    #[clap(long, env)]
+    api_key: Option<String>,
+
     #[clap(long, env)]
     watermark_gamma: Option<f32>,
     #[clap(long, env)]
@@ -439,6 +644,17 @@ struct Args {
     /// Control the maximum number of inputs that a client can send in a single request
     #[clap(default_value = "4", long, env)]
     max_client_batch_size: usize,
+
+    /// Lora Adapters a list of adapter ids i.e. `repo/adapter1,repo/adapter2` to load during
+    /// startup that will be available to callers via the `adapter_id` field in a request.
+    #[clap(long, env)]
+    lora_adapters: Option<String>,
+
+    /// Control if anonymous usage stats are collected.
+    /// Options are "on", "off" and "no-stack"
+    /// Defaul is on.
+    #[clap(default_value = "on", long, env)]
+    usage_stats: UsageStatsLevel,
 }
 
 #[derive(Debug)]
@@ -471,7 +687,10 @@ fn shard_manager(
     rope_factor: Option<f32>,
     max_total_tokens: usize,
     max_batch_size: Option<usize>,
+    max_input_tokens: usize,
+    lora_adapters: Option<String>,
     otlp_endpoint: Option<String>,
+    otlp_service_name: String,
     log_level: LevelFilter,
     status_sender: mpsc::Sender<ShardStatus>,
     shutdown: Arc<AtomicBool>,
@@ -521,12 +740,13 @@ fn shard_manager(
 
     if let Some(dtype) = dtype {
         shard_args.push("--dtype".to_string());
-        shard_args.push(dtype.to_string());
+        shard_args.push(dtype.to_string())
     }
+
     // Model optional revision
     if let Some(revision) = revision {
         shard_args.push("--revision".to_string());
-        shard_args.push(revision);
+        shard_args.push(revision)
     }
 
     let rope = match (rope_scaling, rope_factor) {
@@ -536,12 +756,20 @@ fn shard_manager(
         (None, Some(factor)) => Some((RopeScaling::Linear, factor)),
     };
 
-    // OpenTelemetry
+    // OpenTelemetry Endpoint
     if let Some(otlp_endpoint) = otlp_endpoint {
         shard_args.push("--otlp-endpoint".to_string());
         shard_args.push(otlp_endpoint);
     }
 
+    // OpenTelemetry Service Name
+    shard_args.push("--otlp-service-name".to_string());
+    shard_args.push(otlp_service_name);
+
+    // In case we use sliding window, we may ignore the sliding in flash for some backends depending on the parameter.
+    shard_args.push("--max-input-tokens".to_string());
+    shard_args.push(max_input_tokens.to_string());
+
     // Copy current process env
     let mut envs: Vec<(OsString, OsString)> = env::vars_os().collect();
 
@@ -598,6 +826,11 @@ fn shard_manager(
         envs.push(("MAX_BATCH_SIZE".into(), max_batch_size.to_string().into()));
     }
 
+    // Lora Adapters
+    if let Some(lora_adapters) = lora_adapters {
+        envs.push(("LORA_ADAPTERS".into(), lora_adapters.into()));
+    }
+
     // If huggingface_hub_cache is some, pass it to the shard
     // Useful when running inside a docker container
     if let Some(huggingface_hub_cache) = huggingface_hub_cache {
@@ -647,6 +880,7 @@ fn shard_manager(
         .args(shard_args)
         .env_clear()
         .envs(envs)
+        .stdin(Stdio::piped())
         .stdout(Stdio::piped())
         .stderr(Stdio::piped())
         .process_group(0)
@@ -668,12 +902,13 @@ fn shard_manager(
     };
 
     // Redirect STDOUT to the console
+    let mut pstdin = p.stdin.take().unwrap();
     let shard_stdout_reader = BufReader::new(p.stdout.take().unwrap());
     let shard_stderr_reader = BufReader::new(p.stderr.take().unwrap());
 
     //stdout tracing thread
     thread::spawn(move || {
-        log_lines(shard_stdout_reader.lines());
+        log_lines(shard_stdout_reader);
     });
     // We read stderr in another thread as it seems that lines() can block in some cases
     let (err_sender, err_receiver) = mpsc::channel();
@@ -682,6 +917,18 @@ fn shard_manager(
             err_sender.send(line).unwrap_or(());
         }
     });
+    // We read stdin in another thread as it seems that lines() can block in some cases
+    thread::spawn(move || {
+        let mut stdin = io::stdin(); // We get `Stdin` here.
+        loop {
+            let mut buffer = vec![0; 4096];
+            if let Ok(n) = stdin.read(&mut buffer) {
+                if n > 0 {
+                    let _ = pstdin.write_all(&buffer[..n]);
+                }
+            }
+        }
+    });
 
     let mut ready = false;
     let start_time = Instant::now();
@@ -738,7 +985,10 @@ fn shutdown_shards(shutdown: Arc<AtomicBool>, shutdown_receiver: &mpsc::Receiver
 fn num_cuda_devices() -> Option<usize> {
     let devices = match env::var("CUDA_VISIBLE_DEVICES") {
         Ok(devices) => devices,
-        Err(_) => env::var("NVIDIA_VISIBLE_DEVICES").ok()?,
+        Err(_) => match env::var("NVIDIA_VISIBLE_DEVICES") {
+            Ok(devices) => devices,
+            Err(_) => env::var("ZE_AFFINITY_MASK").ok()?,
+        },
     };
     let n_devices = devices.split(',').count();
     Some(n_devices)
@@ -786,19 +1036,38 @@ impl PythonLogMessage {
     }
 }
 
-impl TryFrom<&String> for PythonLogMessage {
+impl TryFrom<&[u8]> for PythonLogMessage {
     type Error = serde_json::Error;
 
-    fn try_from(value: &String) -> Result<Self, Self::Error> {
-        serde_json::from_str::<Self>(value)
+    fn try_from(value: &[u8]) -> Result<Self, Self::Error> {
+        serde_json::from_slice::<Self>(value)
     }
 }
 
-fn log_lines<S: Sized + BufRead>(lines: Lines<S>) {
-    for line in lines.map_while(Result::ok) {
-        match PythonLogMessage::try_from(&line) {
-            Ok(log) => log.trace(),
-            Err(_) => tracing::debug!("{line}"),
+fn log_lines<R: Sized + Read>(mut bufread: BufReader<R>) {
+    let mut buffer = vec![0u8; 8 * 4096];
+    let mut stdout = std::io::stdout();
+    loop {
+        let n = bufread.read(&mut buffer);
+        if let Ok(n) = n {
+            if n > 0 {
+                let mut lines = buffer[..n].split(|i| *i == b'\n').peekable();
+                while let Some(line) = lines.next() {
+                    match PythonLogMessage::try_from(line) {
+                        Ok(log) => log.trace(),
+                        // For interactive debugging ?
+                        Err(_) => {
+                            if LevelFilter::current() >= tracing::Level::DEBUG {
+                                stdout.write_all(line).unwrap();
+                                if lines.peek().is_some() {
+                                    stdout.write_all(b"\n").unwrap();
+                                }
+                                stdout.flush().unwrap();
+                            }
+                        }
+                    }
+                }
+            }
         }
     }
 }
@@ -811,9 +1080,9 @@ fn find_num_shards(
     let num_shard = match (sharded, num_shard) {
         (Some(true), None) => {
             // try to default to the number of available GPUs
-            tracing::info!("Parsing num_shard from CUDA_VISIBLE_DEVICES/NVIDIA_VISIBLE_DEVICES");
+            tracing::info!("Parsing num_shard from CUDA_VISIBLE_DEVICES/NVIDIA_VISIBLE_DEVICES/ZE_AFFINITY_MASK");
             let n_devices = num_cuda_devices()
-                .expect("--num-shard and CUDA_VISIBLE_DEVICES/NVIDIA_VISIBLE_DEVICES are not set");
+                .expect("--num-shard and CUDA_VISIBLE_DEVICES/NVIDIA_VISIBLE_DEVICES/ZE_AFFINITY_MASK are not set");
             if n_devices <= 1 {
                 return Err(LauncherError::NotEnoughCUDADevices(format!(
                     "`sharded` is true but only found {n_devices} CUDA devices"
@@ -863,13 +1132,20 @@ enum LauncherError {
     WebserverCannotStart,
 }
 
-fn download_convert_model(args: &Args, running: Arc<AtomicBool>) -> Result<(), LauncherError> {
+fn download_convert_model(
+    model_id: &str,
+    revision: Option<&str>,
+    trust_remote_code: bool,
+    huggingface_hub_cache: Option<&str>,
+    weights_cache_override: Option<&str>,
+    running: Arc<AtomicBool>,
+) -> Result<(), LauncherError> {
     // Enter download tracing span
     let _span = tracing::span!(tracing::Level::INFO, "download").entered();
 
     let mut download_args = vec![
         "download-weights".to_string(),
-        args.model_id.to_string(),
+        model_id.to_string(),
         "--extension".to_string(),
         ".safetensors".to_string(),
         "--logger-level".to_string(),
@@ -878,13 +1154,13 @@ fn download_convert_model(args: &Args, running: Arc<AtomicBool>) -> Result<(), L
     ];
 
     // Model optional revision
-    if let Some(revision) = &args.revision {
+    if let Some(revision) = &revision {
         download_args.push("--revision".to_string());
         download_args.push(revision.to_string())
     }
 
     // Trust remote code for automatic peft fusion
-    if args.trust_remote_code {
+    if trust_remote_code {
         download_args.push("--trust-remote-code".to_string());
     }
 
@@ -899,7 +1175,7 @@ fn download_convert_model(args: &Args, running: Arc<AtomicBool>) -> Result<(), L
 
     // If huggingface_hub_cache is set, pass it to the download process
     // Useful when running inside a docker container
-    if let Some(ref huggingface_hub_cache) = args.huggingface_hub_cache {
+    if let Some(ref huggingface_hub_cache) = huggingface_hub_cache {
         envs.push(("HUGGINGFACE_HUB_CACHE".into(), huggingface_hub_cache.into()));
     };
 
@@ -917,7 +1193,7 @@ fn download_convert_model(args: &Args, running: Arc<AtomicBool>) -> Result<(), L
 
     // If args.weights_cache_override is some, pass it to the download process
     // Useful when running inside a HuggingFace Inference Endpoint
-    if let Some(weights_cache_override) = &args.weights_cache_override {
+    if let Some(weights_cache_override) = &weights_cache_override {
         envs.push((
             "WEIGHTS_CACHE_OVERRIDE".into(),
             weights_cache_override.into(),
@@ -925,7 +1201,7 @@ fn download_convert_model(args: &Args, running: Arc<AtomicBool>) -> Result<(), L
     };
 
     // Start process
-    tracing::info!("Starting download process.");
+    tracing::info!("Starting check and download process for {model_id}");
     let mut download_process = match Command::new("text-generation-server")
         .args(download_args)
         .env_clear()
@@ -951,7 +1227,7 @@ fn download_convert_model(args: &Args, running: Arc<AtomicBool>) -> Result<(), L
     let download_stdout = BufReader::new(download_process.stdout.take().unwrap());
 
     thread::spawn(move || {
-        log_lines(download_stdout.lines());
+        log_lines(download_stdout);
     });
 
     let download_stderr = BufReader::new(download_process.stderr.take().unwrap());
@@ -967,7 +1243,7 @@ fn download_convert_model(args: &Args, running: Arc<AtomicBool>) -> Result<(), L
     loop {
         if let Some(status) = download_process.try_wait().unwrap() {
             if status.success() {
-                tracing::info!("Successfully downloaded weights.");
+                tracing::info!("Successfully downloaded weights for {model_id}");
                 break;
             }
 
@@ -1001,6 +1277,8 @@ fn spawn_shards(
     args: &Args,
     cuda_graphs: Vec<usize>,
     max_total_tokens: usize,
+    max_input_tokens: usize,
+    quantize: Option<Quantization>,
     max_log_level: LevelFilter,
     shutdown: Arc<AtomicBool>,
     shutdown_receiver: &mpsc::Receiver<()>,
@@ -1021,7 +1299,7 @@ fn spawn_shards(
         let shutdown = shutdown.clone();
         let shutdown_sender = shutdown_sender.clone();
         let otlp_endpoint = args.otlp_endpoint.clone();
-        let quantize = args.quantize;
+        let otlp_service_name = args.otlp_service_name.clone();
         let speculate = args.speculate;
         let dtype = args.dtype;
         let trust_remote_code = args.trust_remote_code;
@@ -1034,6 +1312,7 @@ fn spawn_shards(
         let rope_scaling = args.rope_scaling;
         let rope_factor = args.rope_factor;
         let max_batch_size = args.max_batch_size;
+        let lora_adapters = args.lora_adapters.clone();
         thread::spawn(move || {
             shard_manager(
                 model_id,
@@ -1058,7 +1337,10 @@ fn spawn_shards(
                 rope_factor,
                 max_total_tokens,
                 max_batch_size,
+                max_input_tokens,
+                lora_adapters,
                 otlp_endpoint,
+                otlp_service_name,
                 max_log_level,
                 status_sender,
                 shutdown,
@@ -1153,6 +1435,10 @@ fn spawn_webserver(
         args.model_id,
     ];
 
+    // Pass usage stats flags to router
+    router_args.push("--usage-stats".to_string());
+    router_args.push(args.usage_stats.to_string());
+
     // Grammar support
     if args.disable_grammar_support {
         router_args.push("--disable-grammar-support".to_string());
@@ -1192,12 +1478,22 @@ fn spawn_webserver(
         router_args.push(otlp_endpoint);
     }
 
+    // OpenTelemetry
+    let otlp_service_name = args.otlp_service_name;
+    router_args.push("--otlp-service-name".to_string());
+    router_args.push(otlp_service_name);
+
     // CORS origins
     for origin in args.cors_allow_origin.into_iter() {
         router_args.push("--cors-allow-origin".to_string());
         router_args.push(origin);
     }
 
+    // API Key
+    if let Some(api_key) = args.api_key {
+        router_args.push("--api-key".to_string());
+        router_args.push(api_key);
+    }
     // Ngrok
     if args.ngrok {
         router_args.push("--ngrok".to_string());
@@ -1286,16 +1582,6 @@ fn terminate(process_name: &str, mut process: Child, timeout: Duration) -> io::R
 }
 
 fn main() -> Result<(), LauncherError> {
-    match Command::new("ldconfig").spawn() {
-        Ok(_) => {}
-        Err(err) => {
-            tracing::warn!(
-                "Unable to refresh ldconfig cache. Skipping (useless in most cases). Details {:?}",
-                err
-            )
-        }
-    }
-
     // Pattern match configuration
     let args: Args = Args::parse();
 
@@ -1336,34 +1622,12 @@ fn main() -> Result<(), LauncherError> {
 
     tracing::info!("{:#?}", args);
 
-    let get_max_position_embeddings = || -> Result<usize, Box<dyn std::error::Error>> {
-        let model_id = args.model_id.clone();
-        let mut path = std::path::Path::new(&args.model_id).to_path_buf();
-        let filename = if !path.exists() {
-            // Assume it's a hub id
-            let api = Api::new()?;
-            let repo = if let Some(ref revision) = args.revision {
-                api.repo(Repo::with_revision(
-                    model_id,
-                    RepoType::Model,
-                    revision.to_string(),
-                ))
-            } else {
-                api.model(model_id)
-            };
-            repo.get("config.json")?
-        } else {
-            path.push("config.json");
-            path
-        };
-
-        let content = std::fs::read_to_string(filename)?;
-        let config: RawConfig = serde_json::from_str(&content)?;
-        let config: Config = config.into();
-
-        // Quantization usually means you're even more RAM constrained.
-        let max_default = 4096;
+    let config: Option<Config> = get_config(&args.model_id, &args.revision).ok();
+    let quantize = config.as_ref().and_then(|c| c.quantize);
+    // Quantization usually means you're even more RAM constrained.
+    let max_default = 4096;
 
+    let max_position_embeddings = if let Some(config) = &config {
         if let Some(max_position_embeddings) = config.max_position_embeddings {
             if max_position_embeddings > max_default {
                 let max = max_position_embeddings;
@@ -1373,17 +1637,20 @@ fn main() -> Result<(), LauncherError> {
                 {
                     tracing::info!("Model supports up to {max} but tgi will now set its default to {max_default} instead. This is to save VRAM by refusing large prompts in order to allow more users on the same hardware. You can increase that size using `--max-batch-prefill-tokens={} --max-total-tokens={max} --max-input-tokens={}`.", max + 50, max - 1);
                 }
-                Ok(max_default)
+                max_default
             } else {
-                Ok(max_position_embeddings)
+                max_position_embeddings
             }
         } else {
-            Err(Box::new(LauncherError::ArgumentValidation(
-                "no max defined".to_string(),
-            )))
+            max_default
         }
+    } else {
+        max_default
     };
-    let max_position_embeddings: usize = get_max_position_embeddings().unwrap_or(4096);
+    let (prefix_caching, attention) = resolve_attention(&config, &args.lora_adapters);
+    tracing::info!("Using attention {attention} - Prefix caching {prefix_caching}");
+    std::env::set_var("USE_PREFIX_CACHING", prefix_caching);
+    std::env::set_var("ATTENTION", attention);
 
     let max_input_tokens = {
         match (args.max_input_tokens, args.max_input_length) {
@@ -1440,18 +1707,26 @@ fn main() -> Result<(), LauncherError> {
         )));
     }
 
-    let cuda_graphs = match (&args.cuda_graphs, &args.quantize) {
+    if matches!(args.quantize, Some(Quantization::Bitsandbytes)) {
+        tracing::warn!("Bitsandbytes is deprecated, use `eetq` instead, which provides better latencies overall and is drop-in in most cases.");
+    }
+    let quantize = args.quantize.or(quantize);
+    let cuda_graphs = match (&args.cuda_graphs, &quantize) {
         (Some(cuda_graphs), _) => cuda_graphs.iter().cloned().filter(|&c| c > 0).collect(),
         #[allow(deprecated)]
         (
             None,
             Some(
                 Quantization::Bitsandbytes
-                | Quantization::BitsandbytesNF4
-                | Quantization::BitsandbytesFP4,
+                | Quantization::BitsandbytesNf4
+                | Quantization::BitsandbytesFp4,
             ),
         ) => {
-            tracing::info!("Bitsandbytes doesn't work with cuda graphs, deactivating them");
+            tracing::warn!("Bitsandbytes doesn't work with cuda graphs, deactivating them");
+            vec![]
+        }
+        (None, Some(Quantization::Exl2)) => {
+            tracing::warn!("Exl2 doesn't work with cuda graphs, deactivating them");
             vec![]
         }
         _ => {
@@ -1475,6 +1750,11 @@ fn main() -> Result<(), LauncherError> {
 
     let num_shard = find_num_shards(args.sharded, args.num_shard)?;
     if num_shard > 1 {
+        if matches!(args.quantize, Some(Quantization::Exl2)) {
+            return Err(LauncherError::ArgumentValidation(
+                "Sharding is currently not supported with `exl2` quantization".into(),
+            ));
+        }
         tracing::info!("Sharding model on {num_shard} processes");
     }
 
@@ -1516,7 +1796,55 @@ fn main() -> Result<(), LauncherError> {
     .expect("Error setting Ctrl-C handler");
 
     // Download and convert model weights
-    download_convert_model(&args, running.clone())?;
+    download_convert_model(
+        &args.model_id,
+        args.revision.as_deref(),
+        args.trust_remote_code,
+        args.huggingface_hub_cache.as_deref(),
+        args.weights_cache_override.as_deref(),
+        running.clone(),
+    )?;
+
+    // Download and convert lora adapters if any
+    if let Some(lora_adapters) = &args.lora_adapters {
+        for adapter in lora_adapters.split(',') {
+            // skip download if a path is provided
+            if adapter.contains('=') {
+                continue;
+            }
+
+            let adapter = adapter.trim();
+
+            // check if adapter has more than 1 '@'
+            if adapter.matches('@').count() > 1 {
+                return Err(LauncherError::ArgumentValidation(format!(
+                    "Invalid LoRA adapter format: {}",
+                    adapter
+                )));
+            }
+
+            // capture adapter_id, path, revision in format of adapter_id=path@revision
+            let re = Regex::new(r"^([^=@]+)(?:=([^@]+))?(?:@(.+))?$").unwrap();
+            if let Some(caps) = re.captures(adapter) {
+                let adapter_id = caps.get(1).map_or("", |m| m.as_str());
+                let revision = caps.get(3).map(|m| m.as_str());
+
+                download_convert_model(
+                    adapter_id,
+                    revision,
+                    args.trust_remote_code,
+                    args.huggingface_hub_cache.as_deref(),
+                    args.weights_cache_override.as_deref(),
+                    running.clone(),
+                )?;
+            } else {
+                return Err(LauncherError::ArgumentValidation(format!(
+                    "Invalid LoRA adapter format: {}",
+                    adapter
+                )));
+            }
+        }
+    }
 
     if !running.load(Ordering::SeqCst) {
         // Launcher was asked to stop
@@ -1537,6 +1865,8 @@ fn main() -> Result<(), LauncherError> {
         &args,
         cuda_graphs,
         max_total_tokens,
+        max_input_tokens,
+        quantize,
         max_log_level,
         shutdown.clone(),
         &shutdown_receiver,
@@ -1561,9 +1891,8 @@ fn main() -> Result<(), LauncherError> {
         shutdown.clone(),
         &shutdown_receiver,
     )
-    .map_err(|err| {
+    .inspect_err(|_| {
         shutdown_shards(shutdown.clone(), &shutdown_receiver);
-        err
     })?;
 
     // Default exit code
diff --git a/load_tests/common.js b/load_tests/common.js
index e0a10595..d890bf67 100644
--- a/load_tests/common.js
+++ b/load_tests/common.js
@@ -33,13 +33,13 @@ export function get_options() {
             //     rate: 20,
             //     timeUnit: '1s',
             // },
-            load_test: {
-                executor: 'constant-arrival-rate',
-                duration: '60s',
-                preAllocatedVUs: 100,
-                rate: 1,
-                timeUnit: '1s',
-            },
+            // load_test: {
+            //     executor: 'constant-arrival-rate',
+            //     duration: '60s',
+            //     preAllocatedVUs: 100,
+            //     rate: 1,
+            //     timeUnit: '1s',
+            // },
             // breakpoint: {
             //     executor: 'ramping-arrival-rate', //Assure load increase if the system slows
             //     preAllocatedVUs: 300,
@@ -47,12 +47,12 @@ export function get_options() {
             //         { duration: '60s', target: 100 }, // just slowly ramp-up to a HUGE load
             //     ],
             // },
-            // throughput: {
-            //     executor: 'shared-iterations',
-            //     vus: 100,
-            //     iterations: 200,
-            //     maxDuration: '40s',
-            // },
+            throughput: {
+                executor: 'shared-iterations',
+                vus: 100,
+                iterations: 200,
+                maxDuration: '40s',
+            },
         },
     };
 }
diff --git a/load_tests/orca.py b/load_tests/orca.py
index e607d27c..e445afd5 100644
--- a/load_tests/orca.py
+++ b/load_tests/orca.py
@@ -20,7 +20,7 @@ def main():
             break
 
     with open("./small.json", "w") as f:
-        data = json.dump(conversations, f, indent=4)
+        json.dump(conversations, f, indent=4)
 
 
 if __name__ == "__main__":
diff --git a/nix/client.nix b/nix/client.nix
new file mode 100644
index 00000000..351fd08a
--- /dev/null
+++ b/nix/client.nix
@@ -0,0 +1,21 @@
+{
+  buildPythonPackage,
+  poetry-core,
+  huggingface-hub,
+  pydantic,
+}:
+
+buildPythonPackage {
+  name = "text-generation";
+
+  src = ../clients/python;
+
+  pyproject = true;
+
+  build-system = [ poetry-core ];
+
+  dependencies = [
+    huggingface-hub
+    pydantic
+  ];
+}
diff --git a/nix/crate-overrides.nix b/nix/crate-overrides.nix
new file mode 100644
index 00000000..a4e74c6d
--- /dev/null
+++ b/nix/crate-overrides.nix
@@ -0,0 +1,87 @@
+{ pkgs, nix-filter }:
+
+let
+  filter = nix-filter.lib;
+in
+with pkgs;
+defaultCrateOverrides
+// {
+  aws-lc-rs = attrs: {
+    # aws-lc-rs does its own custom parsing of Cargo environment
+    # variables like DEP_.*_INCLUDE. However buildRustCrate does
+    # not use the version number, so the parsing fails.
+    postPatch = ''
+      substituteInPlace build.rs \
+        --replace-fail \
+        "assert!(!selected.is_empty()" \
+        "// assert!(!selected.is_empty()"
+    '';
+  };
+  rav1e = attrs: { env.CARGO_ENCODED_RUSTFLAGS = "-C target-feature=-crt-static"; };
+
+  grpc-metadata = attrs: {
+    src = filter {
+      root = ../backends/grpc-metadata;
+      include = with filter; [
+        isDirectory
+        (matchExt "rs")
+      ];
+    };
+  };
+  pyo3-build-config = attrs: {
+    buildInputs = [ python3 ];
+  };
+  text-generation-benchmark = attrs: {
+    src = filter {
+      root = ../benchmark;
+      include = with filter; [
+        isDirectory
+        (matchExt "rs")
+      ];
+    };
+  };
+  text-generation-client = attrs: {
+    src = filter {
+      root = ../.;
+      include = with filter; [
+        isDirectory
+        (and (inDirectory "backends/client") (matchExt "rs"))
+        (and (inDirectory "proto") (matchExt "proto"))
+      ];
+    };
+    postPatch = "cd backends/client";
+    buildInputs = [ protobuf ];
+  };
+  text-generation-launcher = attrs: {
+    src = filter {
+      root = ../launcher;
+      include = with filter; [
+        isDirectory
+        (matchExt "rs")
+      ];
+    };
+  };
+  text-generation-router = attrs: {
+    src = filter {
+      root = ../router;
+      include = with filter; [
+        isDirectory
+        (matchExt "rs")
+      ];
+    };
+  };
+  text-generation-router-v3 = attrs: {
+    # We need to do the src/source root dance so that the build
+    # has access to the protobuf file.
+    src = filter {
+      root = ../.;
+      include = with filter; [
+        isDirectory
+        (and (inDirectory "backends/v3") (matchExt "rs"))
+        (and (inDirectory "proto") (matchExt "proto"))
+      ];
+    };
+    postPatch = "cd backends/v3";
+    buildInputs = [ protobuf ];
+  };
+}
diff --git a/nix/docker.nix b/nix/docker.nix
new file mode 100644
index 00000000..c4b1d899
--- /dev/null
+++ b/nix/docker.nix
@@ -0,0 +1,23 @@
+{
+  dockerTools,
+  cacert,
+  text-generation-inference,
+  stream ? false,
+}:
+
+let
+  build = if stream then dockerTools.streamLayeredImage else dockerTools.buildLayeredImage;
+in
+build {
+  name = "tgi-docker";
+  tag = "latest";
+  config = {
+    EntryPoint = [ "${text-generation-inference}/bin/text-generation-inference" ];
+    Env = [
+      "HF_HOME=/data"
+      "PORT=80"
+    ];
+
+  };
+  contents = [ cacert ];
+}
diff --git a/nix/impure-shell.nix b/nix/impure-shell.nix
new file mode 100644
index 00000000..a4dad4ba
--- /dev/null
+++ b/nix/impure-shell.nix
@@ -0,0 +1,54 @@
+{
+  mkShell,
+  openssl,
+  pkg-config,
+  protobuf,
+  python3,
+  pyright,
+  redocly,
+  ruff,
+  rust-bin,
+  server,
+}:
+
+mkShell {
+  buildInputs =
+    [
+      openssl.dev
+      pkg-config
+      (rust-bin.stable.latest.default.override {
+        extensions = [
+          "rust-analyzer"
+          "rust-src"
+        ];
+      })
+      protobuf
+      pyright
+      redocly
+      ruff
+    ]
+    ++ (with python3.pkgs; [
+      venvShellHook
+      docker
+      pip
+      ipdb
+      click
+      pytest
+      pytest-asyncio
+      syrupy
+    ]);
+
+  inputsFrom = [ server ];
+
+  venvDir = "./.venv";
+
+  postVenvCreation = ''
+    unset SOURCE_DATE_EPOCH
+    ( cd server ; python -m pip install --no-dependencies -e . )
+    ( cd clients/python ; python -m pip install --no-dependencies -e . )
+  '';
+  postShellHook = ''
+    unset SOURCE_DATE_EPOCH
+    export PATH=$PATH:~/.cargo/bin
+  '';
+}
diff --git a/nix/server.nix b/nix/server.nix
new file mode 100644
index 00000000..7406d563
--- /dev/null
+++ b/nix/server.nix
@@ -0,0 +1,115 @@
+{
+  nix-filter,
+  buildPythonPackage,
+  poetry-core,
+  mypy-protobuf,
+  awq-inference-engine,
+  causal-conv1d,
+  eetq,
+  einops,
+  exllamav2,
+  fbgemm-gpu,
+  flashinfer,
+  flash-attn,
+  flash-attn-layer-norm,
+  flash-attn-rotary,
+  flash-attn-v1,
+  grpc-interceptor,
+  grpcio-reflection,
+  grpcio-status,
+  grpcio-tools,
+  hf-transfer,
+  loguru,
+  mamba-ssm,
+  marlin-kernels,
+  moe-kernels,
+  opentelemetry-api,
+  opentelemetry-exporter-otlp,
+  opentelemetry-instrumentation-grpc,
+  opentelemetry-semantic-conventions,
+  peft,
+  punica-kernels,
+  safetensors,
+  tokenizers,
+  torch,
+  sentencepiece,
+  transformers,
+  typer,
+  vllm,
+}:
+
+let
+  filter = nix-filter.lib;
+in
+buildPythonPackage {
+  name = "text-generation-server";
+
+  src = filter {
+    root = ../.;
+    include = with filter; [
+      isDirectory
+      (and (inDirectory "server") (or_ (matchExt "py") (matchExt "pyi")))
+      "server/pyproject.toml"
+      (and (inDirectory "proto/v3") (matchExt "proto"))
+    ];
+  };
+
+  pyproject = true;
+
+  build-system = [ poetry-core ];
+
+  nativeBuildInputs = [ mypy-protobuf ];
+
+  pythonRelaxDeps = [
+    "einops"
+    "huggingface-hub"
+    "loguru"
+    "opentelemetry-instrumentation-grpc"
+    "sentencepiece"
+    "typer"
+  ];
+
+  pythonRemoveDeps = [ "scipy" ];
+
+  dependencies = [
+    awq-inference-engine
+    eetq
+    causal-conv1d
+    einops
+    exllamav2
+    fbgemm-gpu
+    flashinfer
+    flash-attn
+    flash-attn-layer-norm
+    flash-attn-rotary
+    grpc-interceptor
+    grpcio-reflection
+    grpcio-status
+    grpcio-tools
+    hf-transfer
+    loguru
+    mamba-ssm
+    marlin-kernels
+    moe-kernels
+    opentelemetry-api
+    opentelemetry-exporter-otlp
+    opentelemetry-instrumentation-grpc
+    opentelemetry-semantic-conventions
+    peft
+    punica-kernels
+    safetensors
+    sentencepiece
+    tokenizers
+    transformers
+    typer
+    vllm
+  ];
+
+  prePatch = ''
+    python -m grpc_tools.protoc -Iproto/v3 --python_out=server/text_generation_server/pb \
+           --grpc_python_out=server/text_generation_server/pb --mypy_out=server/text_generation_server/pb proto/v3/generate.proto
+    find server/text_generation_server/pb/ -type f -name "*.py" -print0 -exec sed -i -e 's/^\(import.*pb2\)/from . \1/g' {} \;
+    touch server/text_generation_server/pb/__init__.py
+    cd server
+  '';
+}
diff --git a/proto/generate.proto b/proto/generate.proto
index 9921faea..7154db0e 100644
--- a/proto/generate.proto
+++ b/proto/generate.proto
@@ -224,10 +224,11 @@ message DecodeResponse {
 
 message WarmupRequest {
     /// Batch to warmup on
-    repeated Batch batches = 1;
+    Batch batch = 1;
     uint32 max_input_length = 2;
     uint32 max_prefill_tokens = 3;
     uint32 max_total_tokens = 4;
+    uint32 max_batch_total_tokens = 5;
 }
 
 message WarmupResponse {
diff --git a/proto/v3/generate.proto b/proto/v3/generate.proto
new file mode 100644
index 00000000..014ee391
--- /dev/null
+++ b/proto/v3/generate.proto
@@ -0,0 +1,270 @@
+syntax = "proto3";
+
+package generate.v3;
+
+service TextGenerationService {
+  /// Model Info
+  rpc Info(InfoRequest) returns (InfoResponse) {}
+  /// Service discovery
+  rpc ServiceDiscovery(ServiceDiscoveryRequest)
+      returns (ServiceDiscoveryResponse) {}
+  /// Empties batch cache
+  rpc ClearCache(ClearCacheRequest) returns (ClearCacheResponse);
+  /// Remove requests from a cached batch
+  rpc FilterBatch(FilterBatchRequest) returns (FilterBatchResponse);
+  /// Warmup the model and compute max cache size
+  rpc Warmup(WarmupRequest) returns (WarmupResponse);
+  /// Prefill batch and decode first token
+  rpc Prefill(PrefillRequest) returns (PrefillResponse);
+  /// Decode token for a list of prefilled batches
+  rpc Decode(DecodeRequest) returns (DecodeResponse);
+  /// Health check
+  rpc Health(HealthRequest) returns (HealthResponse);
+}
+
+message HealthRequest {}
+message HealthResponse {}
+
+/// Empty request
+message InfoRequest {}
+
+message InfoResponse {
+  bool requires_padding = 1;
+  string dtype = 2;
+  string device_type = 3;
+  optional uint32 window_size = 4;
+  uint32 speculate = 5;
+}
+
+/// Empty request
+message ServiceDiscoveryRequest {}
+
+message ServiceDiscoveryResponse {
+  /// Other shards urls
+  repeated string urls = 1;
+}
+
+message ClearCacheRequest {
+  /// Optional batch id
+  optional uint64 id = 1;
+}
+
+/// Empty response
+message ClearCacheResponse {}
+
+message Image {
+  /// Binary image data.
+  bytes data = 1;
+
+  /// Image MIME type.
+  string mimetype = 2;
+}
+
+message InputChunk {
+  oneof chunk {
+    /// Plain text data
+    string text = 1;
+    /// Image data
+    Image image = 2;
+  }
+}
+
+message Input { repeated InputChunk chunks = 1; }
+
+enum GrammarType {
+  GRAMMAR_TYPE_NONE = 0;
+  GRAMMAR_TYPE_JSON = 1;
+  GRAMMAR_TYPE_REGEX = 2;
+}
+
+message NextTokenChooserParameters {
+  /// exponential scaling output probability distribution
+  float temperature = 1;
+  /// restricting to the k highest probability elements
+  uint32 top_k = 2;
+  /// restricting to top tokens summing to prob_cut_off <= prob_cut_off
+  float top_p = 3;
+  /// restricting to top tokens summing to prob_cut_off <= prob_cut_off
+  float typical_p = 4;
+  /// apply sampling on the logits
+  bool do_sample = 5;
+  /// random seed for sampling
+  uint64 seed = 6;
+  /// repetition penalty
+  float repetition_penalty = 7;
+  /// frequency penalty
+  float frequency_penalty = 9;
+  /// token watermarking using "A Watermark for Large Language Models"
+  bool watermark = 8;
+  /// grammar (applied if not empty)
+  string grammar = 10;
+  /// grammar type
+  GrammarType grammar_type = 11;
+}
+
+message StoppingCriteriaParameters {
+  /// Maximum number of generated tokens
+  uint32 max_new_tokens = 1;
+  /// Optional stopping sequences
+  repeated string stop_sequences = 2;
+  /// Ignore end of sequence token
+  /// used for benchmarking
+  bool ignore_eos_token = 3;
+}
+
+message Request {
+  /// Request ID
+  uint64 id = 1;
+  /// The generation context as chunks
+  Input input_chunks = 8;
+  /// The generation context, stringified input_chunks
+  string inputs = 2;
+  /// Context truncation
+  uint32 truncate = 3;
+  /// Next Token Chooser Parameters
+  NextTokenChooserParameters parameters = 4;
+  /// Stopping Criteria Parameters
+  StoppingCriteriaParameters stopping_parameters = 5;
+  /// Return prefill logprobs
+  bool prefill_logprobs = 6;
+  /// Return most likely n tokens
+  uint32 top_n_tokens = 7;
+  /// Paged attention blocks
+  repeated uint32 blocks = 9;
+  /// Paged attention slots
+  repeated uint32 slots = 10;
+  /// LORA adapter index
+  optional string adapter_id = 11;
+  /// Prefix length that can be retrieved from the KV cache.
+  uint32 prefix_len = 12;
+  /// Context truncation
+  bool add_special_tokens = 13;
+}
+
+message Batch {
+  /// Batch ID
+  uint64 id = 1;
+  /// Individual requests
+  repeated Request requests = 2;
+  /// Batch size (==len(requests))
+  uint32 size = 3;
+  /// Maximum number of tokens this batch will grow to
+  uint32 max_tokens = 4;
+  /// Maximum number of Paged Attention blocks
+  uint32 max_blocks = 5;
+}
+
+message CachedBatch {
+  /// Batch ID
+  uint64 id = 1;
+  /// Individual requests ids
+  repeated uint64 request_ids = 2;
+  /// Batch size (==len(requests))
+  uint32 size = 3;
+  /// Maximum number of tokens this batch will grow to
+  uint32 max_tokens = 4;
+}
+
+enum FinishReason {
+  FINISH_REASON_LENGTH = 0;
+  FINISH_REASON_EOS_TOKEN = 1;
+  FINISH_REASON_STOP_SEQUENCE = 2;
+}
+
+message GeneratedText {
+  /// Output
+  string text = 1;
+  /// Number of generated tokens
+  uint32 generated_tokens = 2;
+  /// Finish reason
+  FinishReason finish_reason = 3;
+  /// Seed
+  optional uint64 seed = 4;
+}
+
+message Tokens {
+  /// Token IDs
+  repeated uint32 ids = 1;
+  /// Logprobs
+  repeated float logprobs = 2;
+  /// tokens
+  repeated string texts = 3;
+  /// special
+  repeated bool is_special = 4;
+}
+
+message Generation {
+  /// Request ID
+  uint64 request_id = 1;
+  /// Prefill tokens (optional)
+  Tokens prefill_tokens = 2;
+  Tokens tokens = 3;
+  /// Complete generated text
+  optional GeneratedText generated_text = 4;
+  /// Top tokens
+  repeated Tokens top_tokens = 5;
+}
+
+message FilterBatchRequest {
+  /// Batch ID
+  uint64 batch_id = 1;
+  /// Requests to keep
+  repeated uint64 request_ids = 2;
+}
+
+message FilterBatchResponse {
+  /// Filtered Batch (cached)
+  CachedBatch batch = 1;
+}
+
+message PrefillRequest {
+  /// Batch
+  Batch batch = 1;
+}
+
+message PrefillResponse {
+  /// Generation
+  repeated Generation generations = 1;
+  /// Next batch (cached)
+  optional CachedBatch batch = 2;
+  /// Forward elapsed time in nanoseconds
+  uint64 forward_ns = 3;
+  /// Decode elapsed time in nanoseconds
+  uint64 decode_ns = 4;
+  /// Total elapsed time in nanoseconds
+  uint64 total_ns = 5;
+}
+
+message DecodeRequest {
+  /// Cached batches
+  repeated CachedBatch batches = 1;
+}
+
+message DecodeResponse {
+  /// Decodes
+  repeated Generation generations = 1;
+  /// Next batch (cached)
+  optional CachedBatch batch = 2;
+  /// Forward elapsed time in nanoseconds
+  uint64 forward_ns = 3;
+  /// Decode elapsed time in nanoseconds
+  uint64 decode_ns = 4;
+  /// Total elapsed time in nanoseconds
+  uint64 total_ns = 5;
+  /// Concatenate elapsed time in nanoseconds
+  optional uint64 concat_ns = 6;
+}
+
+message WarmupRequest {
+  /// Batch to warmup on
+  Batch batch = 1;
+  uint32 max_input_length = 2;
+  uint32 max_prefill_tokens = 3;
+  uint32 max_total_tokens = 4;
+  uint32 max_batch_total_tokens = 5;
+}
+
+message WarmupResponse {
+  /// Maximum number of tokens supported by the model
+  optional uint32 max_supported_total_tokens = 1;
+}
diff --git a/router/Cargo.toml b/router/Cargo.toml
index d164183e..83d85327 100644
--- a/router/Cargo.toml
+++ b/router/Cargo.toml
@@ -7,24 +7,18 @@ edition.workspace = true
 authors.workspace = true
 homepage.workspace = true
 
-[lib]
-path = "src/lib.rs"
-
-[[bin]]
-name = "text-generation-router"
-path = "src/main.rs"
-
 [dependencies]
+async-trait = "0.1.74"
 async-stream = "0.3.5"
-axum = { version = "0.6.20", features = ["json"] }
-axum-tracing-opentelemetry = "0.14.1"
-text-generation-client = { path = "client" }
+axum = { version = "0.7", features = ["json"] }
+axum-tracing-opentelemetry = "0.16"
 clap = { version = "4.4.5", features = ["derive", "env"] }
 futures = "0.3.28"
 hf-hub = { workspace = true }
+itertools = "0.10"
 jsonschema = { version = "0.17.1", features = ["draft202012"] }
-metrics = "0.21.1"
-metrics-exporter-prometheus = { version = "0.12.1", features = [] }
+metrics = { workspace = true }
+metrics-exporter-prometheus = { workspace = true }
 nohash-hasher = "0.2.0"
 opentelemetry = { version = "0.20.0", features = ["rt-tokio"] }
 opentelemetry-otlp = "0.13.0"
@@ -33,23 +27,42 @@ reqwest = { version = "0.11.20", features = [] }
 serde = "1.0.188"
 serde_json = "1.0.107"
 thiserror = "1.0.48"
-tokenizers = { workspace = true}
-tokio = { version = "1.32.0", features = ["rt", "rt-multi-thread", "parking_lot", "signal", "sync"] }
+tokenizers = { workspace = true }
+tokio = { version = "1.32.0", features = [
+  "rt",
+  "rt-multi-thread",
+  "parking_lot",
+  "signal",
+  "sync",
+] }
 tokio-stream = "0.1.14"
-tower-http = { version = "0.4.4", features = ["cors"] }
-tracing = "0.1.37"
+tower-http = { version = "0.5.1", features = ["cors"] }
+tracing = "0.1.40"
 tracing-opentelemetry = "0.21.0"
-tracing-subscriber = { version = "0.3.17", features = ["json", "env-filter"] }
-utoipa = { version = "3.5.0", features = ["axum_extras"] }
-utoipa-swagger-ui = { version = "3.1.5", features = ["axum"] }
+tracing-subscriber = { version = "0.3.18", features = ["json", "env-filter"] }
+utoipa = { version = "4.2.0", features = ["axum_extras"] }
+utoipa-swagger-ui = { version = "6.0.0", features = ["axum"] }
 ngrok = { version = "0.13.1", features = ["axum"], optional = true }
-init-tracing-opentelemetry = { version = "0.14.1", features = ["opentelemetry-otlp"] }
-minijinja = { git = "https://github.com/mitsuhiko/minijinja.git", rev = "5cd4efb" }
+init-tracing-opentelemetry = { version = "0.14.1", features = [
+  "opentelemetry-otlp",
+] }
+minijinja = { workspace = true }
+minijinja-contrib = { workspace = true }
 futures-util = "0.3.30"
 regex = "1.10.3"
 once_cell = "1.19.0"
 image = "0.25.1"
-base64 = "0.22.0"
+base64 = { workspace = true }
+sysinfo = "0.30.13"
+uuid = { version = "1.9.1", default-features = false, features = [
+  "v4",
+  "fast-rng",
+  "macro-diagnostics",
+] }
+csv = "1.3.0"
+ureq = "=2.9"
+pyo3 = { workspace = true }
+
 
 [build-dependencies]
 vergen = { version = "8.2.5", features = ["build", "git", "gitcl"] }
@@ -58,3 +71,4 @@ vergen = { version = "8.2.5", features = ["build", "git", "gitcl"] }
 default = ["ngrok"]
 ngrok = ["dep:ngrok"]
 google = []
+kserve = []
diff --git a/router/client/src/client.rs b/router/client/src/client.rs
deleted file mode 100644
index 4a0d44ee..00000000
--- a/router/client/src/client.rs
+++ /dev/null
@@ -1,527 +0,0 @@
-/// Copyright (C) 2024 Habana Labs, Ltd. an Intel Company.
-
-/// Single shard Client
-use crate::pb::generate::v2::text_generation_service_client::TextGenerationServiceClient;
-use crate::pb::generate::v2::*;
-use crate::Result;
-use std::env;
-use rand::{distributions::Uniform, Rng};
-use grpc_metadata::InjectTelemetryContext;
-use std::cmp;
-use std::time::Duration;
-use tonic::transport::{Channel, Uri};
-use tracing::instrument;
-
-/// Text Generation Inference gRPC client
-#[derive(Debug, Clone)]
-pub struct Client {
-    stub: TextGenerationServiceClient<Channel>,
-}
-
-impl Client {
-    /// Returns a client connected to the given url
-    pub async fn connect(uri: Uri) -> Result<Self> {
-        let channel = Channel::builder(uri).connect().await?;
-
-        Ok(Self {
-            stub: TextGenerationServiceClient::new(channel),
-        })
-    }
-
-    /// Returns a client connected to the given unix socket
-    pub async fn connect_uds(path: String) -> Result<Self> {
-        let channel = Channel::from_shared("http://[::]:50051".to_string())
-            .unwrap()
-            .connect_with_connector(tower::service_fn(move |_: Uri| {
-                tokio::net::UnixStream::connect(path.clone())
-            }))
-            .await?;
-
-        Ok(Self {
-            stub: TextGenerationServiceClient::new(channel),
-        })
-    }
-
-    /// Returns a list of uris or unix sockets of all shards
-    #[instrument(skip(self))]
-    pub async fn service_discovery(&mut self) -> Result<Vec<String>> {
-        let request = tonic::Request::new(ServiceDiscoveryRequest {}).inject_context();
-        let response = self.stub.service_discovery(request).await?;
-        let urls = response
-            .into_inner()
-            .urls
-            .into_iter()
-            // Remove unix socket prefix
-            .map(|url| match url.strip_prefix("unix://") {
-                None => url,
-                Some(stripped_url) => stripped_url.to_string(),
-            })
-            .collect();
-        Ok(urls)
-    }
-
-    /// Get model info
-    #[instrument(skip(self))]
-    pub async fn info(&mut self) -> Result<InfoResponse> {
-        let request = tonic::Request::new(InfoRequest {}).inject_context();
-        let response = self.stub.info(request).await?.into_inner();
-        Ok(response)
-    }
-
-    /// Get model health
-    #[instrument(skip(self))]
-    pub async fn health(&mut self) -> Result<HealthResponse> {
-        let request = tonic::Request::new(HealthRequest {}).inject_context();
-        let response = self.stub.health(request).await?.into_inner();
-        Ok(response)
-    }
-
-    /// Clear the past generations cache
-    #[instrument(skip(self))]
-    pub async fn clear_cache(&mut self, batch_id: Option<u64>) -> Result<()> {
-        let request = tonic::Request::new(ClearCacheRequest { id: batch_id }).inject_context();
-        self.stub.clear_cache(request).await?;
-        Ok(())
-    }
-
-    /// Filter a cached batch
-    #[instrument(skip(self))]
-    pub async fn filter_batch(
-        &mut self,
-        batch_id: u64,
-        request_ids: Vec<u64>,
-    ) -> Result<Option<CachedBatch>> {
-        let request = tonic::Request::new(FilterBatchRequest {
-            batch_id,
-            request_ids,
-        })
-        .inject_context();
-        let filtered_batch = self.stub.filter_batch(request).await?.into_inner();
-        Ok(filtered_batch.batch)
-    }
-
-    /// Warmup on a max size batch
-    ///
-    /// Returns the maximum amount of tokens supported by the hardware
-    #[instrument(skip_all)]
-    pub async fn warmup(
-        &mut self,
-        max_input_length: u32,
-        max_prefill_tokens: u32,
-        max_total_tokens: u32,
-        max_batch_size: Option<usize>,
-        model_id: &str
-    ) -> Result<Option<u32>> {
-        let warmup_enabled: bool = env::var("WARMUP_ENABLED").ok().map_or(true, |value| value.to_lowercase() == "true");
-        if !warmup_enabled {
-            return Ok(None);
-        }
-
-        let read_env_var = |key: &str, default: u32| -> u32 {
-            env::var(key).ok().map_or(default, |value| value.parse::<u32>().unwrap())
-        };
-
-        // get all possible batch sizes
-        let decode_bucket_size: u32 = read_env_var("BATCH_BUCKET_SIZE", 8);
-        let max_decode_batch_size: u32 = match max_batch_size {
-            Some(max_batch_size) => max_batch_size as u32,
-            None => decode_bucket_size
-        };
-        let decode_batch_sizes: Vec<u32> = (decode_bucket_size..max_decode_batch_size+1).step_by(decode_bucket_size as usize).collect();
-
-        let prefill_bucket_size: u32 = read_env_var("PREFILL_BATCH_BUCKET_SIZE", 4);
-        let mut max_prefill_batch_size: u32 = max_prefill_tokens / max_input_length;
-        max_prefill_batch_size = cmp::min(max_prefill_batch_size, max_decode_batch_size);
-        let prefill_batch_sizes: Vec<u32> = (prefill_bucket_size..max_prefill_batch_size+1).step_by(prefill_bucket_size as usize).collect();
-
-        // get all possible sequence lengths for prefill
-        let seq_bucket_size: u32 = read_env_var("PAD_SEQUENCE_TO_MULTIPLE_OF", 128);
-        let mut seq_lengths: Vec<u32> = (seq_bucket_size..max_input_length+1).step_by(seq_bucket_size as usize).collect();
-        if let Some(&last) = seq_lengths.last() {
-            if last < max_input_length {
-                seq_lengths.push(max_input_length);
-            }
-        }
-
-        // execute batch for each combination of batch size and sequence length
-        let mut shapes: Vec<(u32, u32)> = Vec::with_capacity(prefill_batch_sizes.len() * seq_lengths.len());
-        for batch_size in &prefill_batch_sizes {
-            for seq_length in &seq_lengths {
-                shapes.push((*batch_size, *seq_length));
-            }
-        }
-
-        let mut batch_counter: u64 = 0;
-        let mut request_counter: u64 = 0;
-        if model_id.contains("llava") {
-            let mut n_tokens = 0;
-            let mut requests = Vec::new();
-            // Create requests
-            while n_tokens < max_prefill_tokens {
-                let truncate = cmp::min(max_input_length, max_prefill_tokens - n_tokens);
-
-                let mut inputs = String::new();
-                inputs.push_str("![](data:image/jpeg;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAUCAIAAAAC64paAAABg2lDQ1BJQ0MgcHJvZmlsZQAAKJF9kT1Iw0AcxV/TSotUROxQxCFDdbKLijjWKhShQqgVWnUwufQLmrQkKS6OgmvBwY/FqoOLs64OroIg+AHi7OCk6CIl/i8ptIjx4Lgf7+497t4BQqvKNDOQADTdMjKppJjLr4rBVwQQwhAERGVm1uckKQ3P8XUPH1/v4jzL+9yfY0AtmAzwicQJVjcs4g3imU2rznmfOMLKskp8Tjxh0AWJH7muuPzGueSwwDMjRjYzTxwhFks9rPQwKxsa8TRxTNV0yhdyLquctzhr1Qbr3JO/MFzQV5a5TnMUKSxiCRJEKGiggiosxGnVSTGRof2kh3/E8UvkUshVASPHAmrQIDt+8D/43a1ZnJp0k8JJoO/Ftj/GgOAu0G7a9vexbbdPAP8zcKV3/bUWMPtJerOrxY6AwW3g4rqrKXvA5Q4QfarLhuxIfppCsQi8n9E35YHhW6B/ze2ts4/TByBLXaVvgINDYLxE2ese7w719vbvmU5/PycecohsjayNAAAACXBIWXMAAC4jAAAuIwF4pT92AAAAB3RJTUUH6AQIEQMnlTSSjwAAABl0RVh0Q29tbWVudABDcmVhdGVkIHdpdGggR0lNUFeBDhcAAAASSURBVDjLY2AYBaNgFIyCoQsABMQAAeRw1DoAAAAASUVORK5CYII=)");
-                inputs.push_str(&"_test ".to_string().repeat(max_input_length as usize));
-
-                requests.push(Request {
-                    id: 0,
-                    // We truncate the input on the server side to be sure that it has the correct size
-                    inputs,
-                    truncate,
-                    // Set sampling parameters to also take these ops into account in the max memory
-                    parameters: Some(NextTokenChooserParameters {
-                        temperature: 0.9,
-                        top_k: 10,
-                        top_p: 0.9,
-                        typical_p: 0.9,
-                        do_sample: false,
-                        seed: 0,
-                        repetition_penalty: 1.2,
-                        frequency_penalty: 0.1,
-                        watermark: true,
-                        grammar: String::new(),
-                        grammar_type: GrammarType::None as i32,
-                    }),
-                    stopping_parameters: Some(StoppingCriteriaParameters {
-                        max_new_tokens: max_total_tokens - truncate,
-                        stop_sequences: vec![],
-                        ignore_eos_token: true,
-                    }),
-                    prefill_logprobs: true,
-                    top_n_tokens: 20,
-                });
-                n_tokens += max_input_length;
-
-                // Check max_batch_size
-                if Some(requests.len()) == max_batch_size {
-                    break;
-                }
-            }
-
-            let mut batches = Vec::new();
-            batches.push(Batch {
-                id: 0,
-                size: requests.len() as u32,
-                requests,
-                max_tokens: 0,
-            });
-
-            let request = tonic::Request::new(WarmupRequest {
-                batches,
-                max_input_length,
-                max_prefill_tokens,
-                max_total_tokens,
-            })
-            .inject_context();
-            let response = self.stub.warmup(request).await?.into_inner();
-            Ok(response.max_supported_total_tokens)
-        } 
-        else {
-            for shape in shapes.iter() {
-                let (batch_size, seq_length) = shape;
-                let mut batches: Vec<Batch> = vec![
-                    self.create_warmup_batch(
-                        *shape,
-                        &mut batch_counter,
-                        &mut request_counter,
-                        max_input_length,
-                        max_total_tokens,
-                        seq_bucket_size,
-                        false,
-                        None,
-                    )
-                ];
-                // if possible, create second batch in order to trigger concatenate operation
-                if *batch_size < max_decode_batch_size {
-                    batches.push(
-                        self.create_warmup_batch(
-                            (1, *seq_length),
-                            &mut batch_counter,
-                            &mut request_counter,
-                            max_input_length,
-                            max_total_tokens,
-                            seq_bucket_size,
-                            false,
-                            None,
-                        )
-                    );
-                }
-
-                let request = tonic::Request::new(WarmupRequest {
-                    batches,
-                    max_input_length,
-                    max_prefill_tokens,
-                    max_total_tokens,
-                }).inject_context();
-                let _response = self.stub.warmup(request).await?.into_inner();
-            }
-
-            // send batches to warmup all possible decode shapes
-            if decode_batch_sizes.len() > 1 {
-                let steps_per_bucket: u32 = if decode_bucket_size <= max_prefill_batch_size {
-                    decode_bucket_size
-                } else {
-                    decode_bucket_size.div_ceil(max_prefill_batch_size)
-                };
-                let max_new_tokens: u32 = 2 * decode_batch_sizes.len() as u32 * steps_per_bucket;
-
-                let mut requests_send: u32 = cmp::min(max_prefill_batch_size, decode_bucket_size);
-                let mut batches: Vec<Batch> = vec![
-                    self.create_warmup_batch(
-                        (requests_send, seq_bucket_size),
-                        &mut batch_counter,
-                        &mut request_counter,
-                        max_input_length,
-                        max_total_tokens,
-                        seq_bucket_size,
-                        false,
-                        Some(max_new_tokens),
-                    )
-                ];
-
-                let get_current_decode_batch_size = |num: u32| -> u32 {
-                    decode_batch_sizes.iter()
-                        .filter(|&&x| x >= num)
-                        .min()
-                        .copied()
-                        .unwrap()
-                };
-
-                let mut current_decode_batch_size: u32 = get_current_decode_batch_size(requests_send);
-                while current_decode_batch_size < max_decode_batch_size {
-                    let distance_to_next_bucket = current_decode_batch_size + decode_bucket_size - requests_send;
-                    let num_requests: u32 = cmp::min(distance_to_next_bucket, max_prefill_batch_size);
-                    batches.push(
-                        self.create_warmup_batch(
-                            (num_requests, seq_bucket_size),
-                            &mut batch_counter,
-                            &mut request_counter,
-                            max_input_length,
-                            max_total_tokens,
-                            seq_bucket_size,
-                            false,
-                            Some(max_new_tokens),
-                        )
-                    );
-
-                    requests_send += num_requests;
-                    current_decode_batch_size = get_current_decode_batch_size(requests_send);
-                }
-
-                let request = tonic::Request::new(WarmupRequest {
-                    batches,
-                    max_input_length,
-                    max_prefill_tokens,
-                    max_total_tokens,
-                }).inject_context();
-                let _response = self.stub.warmup(request).await?.into_inner();
-            }
-
-            // send batches with default params to warm up Greedy search
-            let mut greedy_shapes: Vec<(u32, u32)> = Vec::with_capacity(prefill_batch_sizes.len());
-            for batch_size in &prefill_batch_sizes {
-                greedy_shapes.push((*batch_size, seq_bucket_size.clone()));
-            }
-            for greedy_shape in greedy_shapes.iter() {
-                let batches: Vec<Batch> = vec![
-                    self.create_warmup_batch(
-                        *greedy_shape,
-                        &mut batch_counter,
-                        &mut request_counter,
-                        max_input_length,
-                        max_total_tokens,
-                        seq_bucket_size,
-                        true,
-                        None,
-                    )
-                ];
-                let request = tonic::Request::new(WarmupRequest {
-                    batches,
-                    max_input_length,
-                    max_prefill_tokens,
-                    max_total_tokens,
-                }).inject_context();
-                let _response = self.stub.warmup(request).await?.into_inner();
-            }
-            Ok(None) // No support for maximum total tokens
-        }
-    }
-
-    #[instrument(skip_all)]
-    fn create_warmup_batch(
-        &mut self,
-        shape: (u32, u32),
-        batch_counter: &mut u64,
-        request_counter: &mut u64,
-        max_input_length: u32,
-        max_total_tokens: u32,
-        seq_bucket_size: u32,
-        default_params: bool,
-        max_new_tokens: Option<u32>,
-    ) -> Batch {
-        *batch_counter += 1;
-        let (batch_size, input_length) = shape;
-        let mut requests = Vec::new();
-        for _ in 0..batch_size {
-            *request_counter += 1;
-            let req_params = if default_params {
-                Some(NextTokenChooserParameters {
-                    temperature: 1.0,
-                    top_k: 0,
-                    top_p: 1.0,
-                    typical_p: 1.0,
-                    do_sample: false,
-                    seed: 0,
-                    repetition_penalty: 1.0,
-                    frequency_penalty: 0.0,
-                    watermark: false,
-                    grammar: String::new(),
-                    grammar_type: GrammarType::None as i32,
-                })
-            } else {
-                Some(NextTokenChooserParameters {
-                    temperature: 0.9,
-                    top_k: 10,
-                    top_p: 0.9,
-                    typical_p: 0.9,
-                    do_sample: true,
-                    seed: 0,
-                    repetition_penalty: 1.2,
-                    frequency_penalty: 0.1,
-                    watermark: false,
-                    grammar: String::new(),
-                    grammar_type: GrammarType::None as i32,
-                })
-            };
-            requests.push(Request {
-                id: *request_counter,
-                inputs: self.get_random_input(input_length, seq_bucket_size),
-                truncate: max_input_length,
-                parameters: req_params,
-                stopping_parameters: Some(StoppingCriteriaParameters {
-                    max_new_tokens: cmp::min(max_new_tokens.unwrap_or(10), max_total_tokens - max_input_length),
-                    stop_sequences: vec![],
-                    ignore_eos_token: true,
-                }),
-                prefill_logprobs: false,
-                top_n_tokens: 0,
-            });
-        }
-
-        Batch {
-            id: *batch_counter,
-            size: requests.len() as u32,
-            requests,
-            max_tokens: max_total_tokens,
-        }
-    }
-
-    #[instrument(skip_all)]
-    fn get_random_input(
-        &mut self,
-        input_length: u32,
-        seq_bucket_size: u32,
-    ) -> String {
-        let skip_tokenizer_in_tgi: bool = env::var("SKIP_TOKENIZER_IN_TGI")
-            .ok()
-            .map_or(false, |value| value.to_lowercase() == "true");
-        if skip_tokenizer_in_tgi {
-            // generate random tokens
-            let mut rng = rand::thread_rng();
-            let range = Uniform::new(2, 8192);
-            let tokens = if input_length % seq_bucket_size == 0 {
-                input_length - seq_bucket_size / 2
-            } else {
-                input_length - (input_length % seq_bucket_size) / 2
-            };
-            (0..tokens)
-                .map(|_| rng.sample(&range).to_string())
-                .collect::<Vec<String>>()
-                .join(", ")
-        } else {
-            // repeat test string to get expected input shape
-            let mut bucket_id = input_length / seq_bucket_size;
-            if input_length % seq_bucket_size != 0 {
-                bucket_id += 1
-            }
-            let repeats = cmp::max(1, (bucket_id - 1) * seq_bucket_size / 2);
-            "_test ".to_string().repeat(repeats as usize)
-        }
-    }
-
-    /// Generate one token for each request in the given batch
-    ///
-    /// Returns Generation for each request in batch
-    /// and the next cached batch
-    #[instrument(skip_all, fields(id = &batch.id, size = &batch.size))]
-    pub async fn prefill(
-        &mut self,
-        batch: Batch,
-    ) -> Result<(Vec<Generation>, Option<CachedBatch>, PrefillTimings)> {
-        let request = tonic::Request::new(PrefillRequest { batch: Some(batch) }).inject_context();
-        let response = self.stub.prefill(request).await?.into_inner();
-        Ok((
-            response.generations,
-            response.batch,
-            PrefillTimings::new(response.forward_ns, response.decode_ns, response.total_ns),
-        ))
-    }
-
-    /// Generate one token for each request in the given cached batches
-    ///
-    /// Returns Generation for each request in batches
-    /// and the next cached batch
-    #[instrument(skip_all, fields(size = batches.iter().map(|batch|{batch.size}).sum::<u32>()))]
-    pub async fn decode(
-        &mut self,
-        batches: Vec<CachedBatch>,
-    ) -> Result<(Vec<Generation>, Option<CachedBatch>, DecodeTimings)> {
-        let request = tonic::Request::new(DecodeRequest { batches }).inject_context();
-        let response = self.stub.decode(request).await?.into_inner();
-        Ok((
-            response.generations,
-            response.batch,
-            DecodeTimings::new(
-                response.concat_ns,
-                response.forward_ns,
-                response.decode_ns,
-                response.total_ns,
-            ),
-        ))
-    }
-}
-
-pub struct PrefillTimings {
-    pub forward: Duration,
-    pub decode: Duration,
-    pub total: Duration,
-}
-
-impl PrefillTimings {
-    fn new(forward_ns: u64, decode_ns: u64, total_ns: u64) -> Self {
-        Self {
-            forward: Duration::from_nanos(forward_ns),
-            decode: Duration::from_nanos(decode_ns),
-            total: Duration::from_nanos(total_ns),
-        }
-    }
-}
-
-pub struct DecodeTimings {
-    pub concat: Option<Duration>,
-    pub forward: Duration,
-    pub decode: Duration,
-    pub total: Duration,
-}
-
-impl DecodeTimings {
-    fn new(concat_ns: Option<u64>, forward_ns: u64, decode_ns: u64, total_ns: u64) -> Self {
-        Self {
-            concat: concat_ns.map(Duration::from_nanos),
-            forward: Duration::from_nanos(forward_ns),
-            decode: Duration::from_nanos(decode_ns),
-            total: Duration::from_nanos(total_ns),
-        }
-    }
-}
diff --git a/router/client/src/lib.rs b/router/client/src/lib.rs
deleted file mode 100644
index 6782d9ff..00000000
--- a/router/client/src/lib.rs
+++ /dev/null
@@ -1,46 +0,0 @@
-//! Text Generation gRPC client library
-
-mod client;
-#[allow(clippy::derive_partial_eq_without_eq)]
-mod pb;
-mod sharded_client;
-
-pub use client::Client;
-pub use pb::generate::v2::HealthResponse;
-pub use pb::generate::v2::InfoResponse as ShardInfo;
-pub use pb::generate::v2::{
-    Batch, CachedBatch, FinishReason, GeneratedText, Generation, GrammarType,
-    NextTokenChooserParameters, Request, StoppingCriteriaParameters, Tokens,
-};
-pub use sharded_client::ShardedClient;
-use thiserror::Error;
-use tonic::transport;
-use tonic::Status;
-
-#[derive(Error, Debug, Clone)]
-pub enum ClientError {
-    #[error("Could not connect to Text Generation server: {0}")]
-    Connection(String),
-    #[error("Server error: {0}")]
-    Generation(String),
-    #[error("Sharded results are empty")]
-    EmptyResults,
-}
-
-impl From<Status> for ClientError {
-    fn from(err: Status) -> Self {
-        let err = Self::Generation(err.message().to_string());
-        tracing::error!("{err}");
-        err
-    }
-}
-
-impl From<transport::Error> for ClientError {
-    fn from(err: transport::Error) -> Self {
-        let err = Self::Connection(err.to_string());
-        tracing::error!("{err}");
-        err
-    }
-}
-
-pub type Result<T> = std::result::Result<T, ClientError>;
diff --git a/router/client/src/pb/.gitignore b/router/client/src/pb/.gitignore
deleted file mode 100644
index 6f5f3d11..00000000
--- a/router/client/src/pb/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-*.rs
diff --git a/router/src/config.rs b/router/src/config.rs
index d27b1136..1a20c40b 100644
--- a/router/src/config.rs
+++ b/router/src/config.rs
@@ -4,9 +4,9 @@ use serde::{Deserialize, Serialize};
 #[serde(tag = "model_type")]
 #[serde(rename_all = "snake_case")]
 pub struct LlavaNext {
-    text_config: TextConfig,
-    vision_config: VisionConfig,
-    image_grid_pinpoints: Vec<(usize, usize)>,
+    pub(crate) text_config: TextConfig,
+    pub(crate) vision_config: VisionConfig,
+    pub(crate) image_grid_pinpoints: Vec<(usize, usize)>,
 }
 
 fn get_anyres_image_grid_shape(
@@ -71,10 +71,12 @@ fn get_unpadded_features(
     let current_aspect_ratio: f64 = current_width as f64 / current_height as f64;
     let (current_height, current_width) = if aspect_ratio > current_aspect_ratio {
         let new_height = (height * current_width) / width;
-        (new_height, current_width)
+        let padding = (current_height - new_height) / 2;
+        (current_height - (2 * padding), current_width)
     } else {
         let new_width = (width * current_height) / height;
-        (current_height, new_width)
+        let padding = (current_width - new_width) / 2;
+        (current_height, current_width - (2 * padding))
     };
 
     let unpadded_features = current_height * current_width;
@@ -88,7 +90,9 @@ impl LlavaNext {
         let patch_size = self.vision_config.patch_size;
         assert!(image_size % patch_size == 0);
         let npatches = image_size / patch_size;
-        let (num_patch_height, num_patch_width) =
+        // Dimensions are intentionally swapped to be bug-compatible with
+        // upstream: https://github.com/LLaVA-VL/LLaVA-NeXT/issues/59
+        let (num_patch_width, num_patch_height) =
             get_anyres_image_grid_shape(height, width, &self.image_grid_pinpoints, image_size);
 
         let (unpadded_features, newline_features) =
@@ -112,20 +116,20 @@ pub struct Idefics2 {}
 
 impl Idefics2 {
     pub fn get_number_of_features(&self, _height: usize, _width: usize) -> usize {
-        320
+        64
     }
 }
 
 #[derive(Clone, Debug, Serialize, Deserialize)]
 #[serde(rename_all = "snake_case")]
 pub struct PaliTextConfig {
-    num_image_tokens: usize,
+    pub(crate) num_image_tokens: usize,
 }
 
 #[derive(Clone, Debug, Serialize, Deserialize)]
 #[serde(rename_all = "snake_case")]
 pub struct Paligemma {
-    text_config: PaliTextConfig,
+    pub(crate) text_config: PaliTextConfig,
 }
 
 impl Paligemma {
@@ -142,6 +146,7 @@ pub enum Config {
     ClipVisionModel(ClipVisionModel),
     Mistral,
     Idefics,
+    Mllama,
     Idefics2(Idefics2),
     Ssm,
     GptBigcode,
@@ -149,15 +154,18 @@ pub enum Config {
     Bloom,
     Mpt,
     Gpt2,
+    Gptj,
     GptNeox,
     Phi,
     #[serde(rename = "phi-msft")]
     PhiMsft,
     Phi3,
+    PhiMoe,
     Llama,
     Baichuan,
     Paligemma(Paligemma),
     Gemma,
+    Gemma2,
     Cohere,
     Drbx,
     Falcon,
@@ -175,8 +183,8 @@ pub struct TextConfig {}
 #[derive(Clone, Debug, Serialize, Deserialize)]
 #[serde(rename_all = "snake_case")]
 pub struct VisionConfig {
-    image_size: usize,
-    patch_size: usize,
+    pub(crate) image_size: usize,
+    pub(crate) patch_size: usize,
 }
 
 #[cfg(test)]
diff --git a/router/src/health.rs b/router/src/health.rs
deleted file mode 100644
index b05b3094..00000000
--- a/router/src/health.rs
+++ /dev/null
@@ -1,72 +0,0 @@
-use std::sync::atomic::{AtomicBool, Ordering};
-use std::sync::Arc;
-use text_generation_client::GrammarType as ProtoGrammarType;
-use text_generation_client::{
-    Batch, NextTokenChooserParameters, Request, ShardedClient, StoppingCriteriaParameters,
-};
-
-// Note: Request ids and batch ids cannot collide.
-const LIVENESS_ID: u64 = u64::MAX;
-const BATCH_ID: u64 = u64::MAX;
-
-#[derive(Clone, Debug)]
-pub(crate) struct Health {
-    client: ShardedClient,
-    generation_health: Arc<AtomicBool>,
-}
-
-impl Health {
-    pub(crate) fn new(client: ShardedClient, generation_health: Arc<AtomicBool>) -> Self {
-        Self {
-            client,
-            generation_health,
-        }
-    }
-
-    pub(crate) async fn check(&mut self) -> bool {
-        if self.generation_health.load(Ordering::SeqCst) {
-            // Generation is healthy, we only check that the shards are answering gRPC calls
-            self.client.health().await.is_ok()
-        } else {
-            // Generation is unhealthy or have not sent any generation request yet
-
-            // Dummy batch of 1 token and 1 generated token
-            let liveness_request = Request {
-                id: LIVENESS_ID,
-                inputs: "liveness".to_string(),
-                truncate: 10,
-                prefill_logprobs: false,
-                parameters: Some(NextTokenChooserParameters {
-                    temperature: 1.0,
-                    top_k: 0,
-                    top_p: 1.0,
-                    typical_p: 1.0,
-                    do_sample: false,
-                    seed: 0,
-                    repetition_penalty: 1.0,
-                    frequency_penalty: 0.0,
-                    watermark: false,
-                    grammar: String::new(),
-                    grammar_type: ProtoGrammarType::None as i32,
-                }),
-                stopping_parameters: Some(StoppingCriteriaParameters {
-                    max_new_tokens: 1,
-                    stop_sequences: vec![],
-                    ignore_eos_token: false,
-                }),
-                top_n_tokens: 0,
-            };
-            let batch = Batch {
-                id: BATCH_ID,
-                requests: vec![liveness_request],
-                size: 1,
-                max_tokens: 2,
-            };
-            // Skips the queue
-            let value = self.client.prefill(batch).await.is_ok();
-            // Update generation health
-            self.generation_health.store(value, Ordering::SeqCst);
-            value
-        }
-    }
-}
diff --git a/router/src/infer.rs b/router/src/infer/chat_template.rs
similarity index 56%
rename from router/src/infer.rs
rename to router/src/infer/chat_template.rs
index 04328246..1071d0ba 100644
--- a/router/src/infer.rs
+++ b/router/src/infer/chat_template.rs
@@ -1,955 +1,111 @@
-/// Copyright (C) 2024 Habana Labs, Ltd. an Intel Company.
-
-/// Batching and inference logic
-use crate::validation::{Validation, ValidationError};
-use crate::{
-    ChatTemplateInputs, ChatTemplateVersions, Entry, GenerateRequest, GenerateStreamResponse,
-    HubTokenizerConfig, Message, MessageChunk, PrefillToken, Queue, Text, TextMessage, Token,
-};
-use crate::{FunctionRef, FunctionsMap, GrammarType, Properties, Tool, ToolType, Tools};
-use futures::future::try_join_all;
+use crate::infer::InferError;
+use crate::{ChatTemplateInputs, Message, MessageChunk, TextMessage, TokenizerConfigToken, Tool};
 use minijinja::{Environment, ErrorKind, Template};
-use nohash_hasher::IntMap;
-use serde_json::{json, Map, Value};
-use std::collections::HashMap;
-use std::sync::{
-    atomic::{AtomicBool, Ordering},
-    Arc,
-};
-use text_generation_client::{
-    Batch, CachedBatch, ClientError, GeneratedText, Generation, ShardedClient, Tokens,
-};
-use thiserror::Error;
-use tokio::sync::mpsc::error::SendError;
-use tokio::sync::{mpsc, Notify, Semaphore, TryAcquireError};
-use tokio::time::Instant;
-use tokio_stream::wrappers::UnboundedReceiverStream;
-use tokio_stream::StreamExt;
-use tracing::{info_span, instrument, Instrument, Span};
-
-/// Inference struct
-#[derive(Clone)]
-pub struct Infer {
-    /// Validation
-    validation: Validation,
-    /// Request queue
-    queue: Queue,
-    /// Shared state
-    shared: Arc<Shared>,
-    /// Chat template
-    chat_template: Option<ChatTemplate>,
-    /// Inference limit
-    limit_concurrent_requests: Arc<Semaphore>,
-}
-
-/// Infer shared state
-struct Shared {
-    /// Batching background Tokio task notifier
-    batching_task: Notify,
-}
+use minijinja_contrib::pycompat;
+use std::collections::HashSet;
 
 /// Raise a exception (custom function) used in the chat templates
-fn raise_exception(err_text: String) -> Result<String, minijinja::Error> {
+pub(crate) fn raise_exception(err_text: String) -> Result<String, minijinja::Error> {
     Err(minijinja::Error::new(ErrorKind::SyntaxError, err_text))
 }
 
-impl Infer {
-    #[allow(clippy::too_many_arguments)]
-    pub(crate) fn new(
-        client: ShardedClient,
-        validation: Validation,
-        waiting_served_ratio: f32,
-        max_batch_prefill_tokens: u32,
-        max_batch_total_tokens: u32,
-        max_waiting_tokens: usize,
-        max_batch_size: Option<usize>,
-        max_concurrent_requests: usize,
-        requires_padding: bool,
-        max_input_length: u32,
-        max_total_tokens: u32,
-        window_size: Option<u32>,
-        speculate: u32,
-        generation_health: Arc<AtomicBool>,
-        tokenizer_config: HubTokenizerConfig,
-    ) -> Self {
-        // Infer shared state
-        let queue = Queue::new(
-            requires_padding,
-            max_input_length,
-            max_total_tokens,
-            16,
-            window_size,
-            speculate
-        );
-        let shared = Arc::new(Shared {
-            batching_task: Notify::new(),
-        });
-
-        // Spawn batching background task that contains all the inference logic
-        tokio::spawn(batching_task(
-            client,
-            waiting_served_ratio,
-            max_batch_prefill_tokens,
-            max_batch_total_tokens,
-            max_waiting_tokens,
-            max_batch_size,
-            queue.clone(),
-            shared.clone(),
-            generation_health,
-        ));
-
-        let chat_template = tokenizer_config
-            .chat_template
-            .and_then(|t| match t {
-                ChatTemplateVersions::Single(template) => Some(template),
-                ChatTemplateVersions::Multiple(templates) => templates
-                    .into_iter()
-                    .find(|t| t.name == "default")
-                    .map(|t| t.template),
-            })
-            .map(|t| {
-                // .strip() is not supported in minijinja
-                let t = t.replace(".strip()", " | trim");
-                ChatTemplate::new(t, tokenizer_config.bos_token, tokenizer_config.eos_token)
-            });
-
-        // Inference limit with a semaphore
-        let semaphore = Arc::new(Semaphore::new(max_concurrent_requests));
-
-        Self {
-            validation,
-            queue,
-            shared,
-            chat_template,
-            limit_concurrent_requests: semaphore,
-        }
-    }
-
-    /// Add a new request to the queue and return a stream of InferStreamResponse
-    #[instrument(skip_all)]
-    pub(crate) async fn generate_stream(
-        &self,
-        request: GenerateRequest,
-    ) -> Result<GenerateStreamResponse, InferError> {
-        // Limit concurrent requests by acquiring a permit from the semaphore
-        let permit = self
-            .clone()
-            .limit_concurrent_requests
-            .try_acquire_owned()
-            .map_err(|err| {
-                metrics::increment_counter!("tgi_request_failure", "err" => "overloaded");
-                tracing::error!("{err}");
-                err
-            })?;
-
-        // Validate request
-        let valid_request = self.validation.validate(request).await.map_err(|err| {
-            metrics::increment_counter!("tgi_request_failure", "err" => "validation");
-            tracing::error!("{err}");
-            err
-        })?;
-
-        // MPSC channel to communicate with the background batching task
-        let (response_tx, response_rx) = mpsc::unbounded_channel();
-        let input_length = valid_request.input_length;
-
-        // Append the request to the queue
-        self.queue.append(Entry {
-            request: valid_request,
-            response_tx,
-            span: Span::current(),
-            temp_span: None,
-            queue_time: Instant::now(),
-            batch_time: None,
-        });
-
-        // Notify the background task that we have a new entry in the queue that needs
-        // to be batched
-        self.shared.batching_task.notify_one();
-
-        // Return stream
-        Ok((
-            permit,
-            input_length,
-            UnboundedReceiverStream::new(response_rx),
-        ))
-    }
-
-    /// Tokenizer the input
-    #[instrument(skip_all)]
-    pub(crate) async fn tokenize(
-        &self,
-        request: GenerateRequest,
-    ) -> Result<Option<tokenizers::Encoding>, InferError> {
-        // Tokenize request
-        let inputs = request.inputs;
-        let truncate = request.parameters.truncate;
-        let encoding = self
-            .validation
-            .tokenize(inputs, truncate)
-            .await
-            .map_err(|err| {
-                tracing::error!("Tokenization {err}");
-                err
-            })?;
-
-        // Return Encoding
-        Ok(encoding.map(|(encoding, _)| encoding))
-    }
-
-    /// Apply the chat template to the chat request
-    #[instrument(skip_all)]
-    pub(crate) fn apply_chat_template(
-        &self,
-        messages: Vec<Message>,
-        grammar_with_prompt: Option<(GrammarType, String)>,
-    ) -> Result<String, InferError> {
-        self.chat_template
-            .as_ref()
-            .ok_or_else(|| InferError::TemplateError(ErrorKind::TemplateNotFound.into()))?
-            .apply(messages, grammar_with_prompt)
-            .map_err(|e| {
-                metrics::increment_counter!("tgi_request_failure", "err" => "template");
-                tracing::error!("{e}");
-                e
-            })
-    }
-
-    /// Add a new request to the queue and return a InferResponse
-    #[instrument(skip_all)]
-    pub(crate) async fn generate(
-        &self,
-        request: GenerateRequest,
-    ) -> Result<InferResponse, InferError> {
-        let use_top_tokens = request.parameters.top_n_tokens.is_some_and(|x| x > 0);
-
-        // Create stream and keep semaphore permit as long as generate lives
-        let (_permit, _input_length, mut stream) = self.generate_stream(request).await?;
-
-        // Return values
-        let mut result_prefill = Vec::new();
-        let mut result_tokens = Vec::new();
-        let mut result_top_tokens = Vec::new();
-        let mut result_generated_text = None;
-        let mut result_start = None;
-        let mut result_queued = None;
-
-        // Iterate on stream
-        while let Some(response) = stream.next().await {
-            match response? {
-                // Add prefill tokens
-                InferStreamResponse::Prefill(tokens) => {
-                    // Create Token objects
-                    // We do that here instead of in the Python code as Rust for loops are faster
-                    result_prefill = tokens
-                        .ids
-                        .into_iter()
-                        .zip(tokens.logprobs.into_iter())
-                        .zip(tokens.texts.into_iter())
-                        .map(|((id, logprob), text)| PrefillToken { id, text, logprob })
-                        .collect();
-                }
-                // Push last token
-                InferStreamResponse::Intermediate { token, top_tokens } => {
-                    result_tokens.push(token);
-                    result_top_tokens.push(top_tokens);
-                }
-                // Final message
-                // Set return values
-                InferStreamResponse::End {
-                    token,
-                    generated_text,
-                    start,
-                    queued,
-                    top_tokens,
-                } => {
-                    result_tokens.push(token);
-                    result_top_tokens.push(top_tokens);
-                    result_generated_text = Some(generated_text);
-                    result_start = Some(start);
-                    result_queued = Some(queued)
-                }
-            }
-        }
-
-        // Check that we received a `InferStreamResponse::End` message
-        if let (Some(generated_text), Some(queued), Some(start)) =
-            (result_generated_text, result_queued, result_start)
-        {
-            Ok(InferResponse {
-                prefill: result_prefill,
-                _input_length,
-                tokens: result_tokens,
-                generated_text,
-                queued,
-                start,
-                top_tokens: if use_top_tokens {
-                    result_top_tokens
-                } else {
-                    Vec::new()
-                },
-            })
-        } else {
-            let err = InferError::IncompleteGeneration;
-            metrics::increment_counter!("tgi_request_failure", "err" => "incomplete");
-            tracing::error!("{err}");
-            Err(err)
-        }
-    }
-    /// Add best_of new requests to the queue and return a InferResponse of the sequence with
-    /// the highest log probability per token
-    #[instrument(skip(self, request))]
-    pub(crate) async fn generate_best_of(
-        &self,
-        request: GenerateRequest,
-        best_of: usize,
-    ) -> Result<(InferResponse, Vec<InferResponse>), InferError> {
-        // validate  best_of parameter separately
-        let best_of = self.validation.validate_best_of(best_of)?;
-
-        // create multiple generate requests
-        let mut infer_responses: Vec<InferResponse> =
-            try_join_all((0..best_of).map(|_| self.generate(request.clone()))).await?;
-
-        // get the sequence with the highest log probability per token
-        let mut max_index = 0;
-        let mut max_logprob: f32 = f32::MIN;
-
-        for (i, response) in infer_responses.iter().enumerate() {
-            // mean logprobs of the generated tokens
-            let sequence_logprob = response
-                .tokens
-                .iter()
-                .map(|token| token.logprob)
-                .sum::<f32>()
-                / response.tokens.len() as f32;
-
-            // set best sequence
-            if sequence_logprob > max_logprob {
-                max_index = i;
-                max_logprob = sequence_logprob;
-            }
-        }
-        let best_response = infer_responses.remove(max_index);
-        Ok((best_response, infer_responses))
-    }
-}
-
 #[derive(Clone)]
-struct ChatTemplate {
+pub(crate) struct ChatTemplate {
     template: Template<'static, 'static>,
     bos_token: Option<String>,
     eos_token: Option<String>,
     use_default_tool_template: bool,
+    variables: HashSet<String>,
 }
 
 impl ChatTemplate {
-    fn new(template: String, bos_token: Option<String>, eos_token: Option<String>) -> Self {
+    pub(crate) fn new(
+        template: String,
+        bos_token: Option<TokenizerConfigToken>,
+        eos_token: Option<TokenizerConfigToken>,
+    ) -> Self {
         let mut env = Box::new(Environment::new());
+        // enable things like .strip() or .capitalize()
+        env.set_unknown_method_callback(pycompat::unknown_method_callback);
         let template_str = template.into_boxed_str();
         env.add_function("raise_exception", raise_exception);
+        tracing::debug!("Loading template: {}", template_str);
 
-        // check if contains the tools variable within the template
-        let use_default_tool_template =
-            !template_str.as_ref().replace(' ', "").contains("{{tools}}");
         // leaking env and template_str as read-only, static resources for performance.
         let template = Box::leak(env)
             .template_from_str(Box::leak(template_str))
             .unwrap();
 
+        // get the list of variables that are used in the template
+        let variables = template.undeclared_variables(true);
+        // check if the `tools` variable is used in the template
+        let use_default_tool_template = !variables.contains("tools");
+        tracing::debug!("Use default tool template: {}", use_default_tool_template);
+
         Self {
             template,
-            bos_token,
-            eos_token,
+            bos_token: bos_token.map(|token| token.as_str().to_string()),
+            eos_token: eos_token.map(|token| token.as_str().to_string()),
             use_default_tool_template,
+            variables,
         }
     }
 
-    fn apply(
+    pub(crate) fn apply(
         &self,
+        guideline: Option<&str>,
         mut messages: Vec<Message>,
-        grammar_with_prompt: Option<(GrammarType, String)>,
+        tools_and_prompt: Option<(Vec<Tool>, String)>,
     ) -> Result<String, InferError> {
-        if self.use_default_tool_template {
-            if let Some(last_message) = messages.last_mut() {
-                if let Some((GrammarType::Json(tools), tool_prompt)) = grammar_with_prompt {
-                    last_message.content.push(MessageChunk::Text(Text {
-                        text: format!("\n---\n{}\n{}", tool_prompt, tools),
-                    }));
-                }
-            }
+        // check if guideline is expected but not provided
+        if self.variables.contains("guideline") && guideline.is_none() {
+            return Err(InferError::MissingTemplateVariable("guideline".to_string()));
         }
 
+        let tools = match tools_and_prompt {
+            Some((tools, tool_prompt)) => {
+                // check if the `tools` variable is used in the template
+                // if not, we need to append the tools to the last message
+                let text = if self.use_default_tool_template {
+                    match serde_json::to_string(&tools) {
+                        Ok(tools_str) => format!("\n---\n{}\n{}", tools_str, tool_prompt),
+                        Err(e) => return Err(InferError::ToolError(e.to_string())),
+                    }
+                } else {
+                    // if the `tools` variable is used in the template, we just append the tool_prompt
+                    format!("\n---\n{}", tool_prompt)
+                };
+                if let Some(last_message) = messages.last_mut() {
+                    last_message.content.push(MessageChunk::Text { text });
+                }
+                Some(tools)
+            }
+            None => None,
+        };
+
         let messages: Vec<TextMessage> = messages.into_iter().map(|c| c.into()).collect();
 
         self.template
             .render(ChatTemplateInputs {
+                guideline,
                 messages,
                 bos_token: self.bos_token.as_deref(),
                 eos_token: self.eos_token.as_deref(),
                 add_generation_prompt: true,
-                tools: None,
-                tools_prompt: None,
+                tools,
             })
             .map_err(InferError::TemplateError)
     }
 }
 
-pub struct ToolGrammar {}
-
-impl ToolGrammar {
-    pub fn apply(
-        tools: Option<Vec<Tool>>,
-        tool_choice: Option<ToolType>,
-    ) -> Result<Option<Tools>, InferError> {
-        if let Some((req_tools, tool_choice)) = tools.zip(tool_choice) {
-            // let tool_prompt = tool_prompt.unwrap_or_default();
-            let tools_to_use = match tool_choice {
-                ToolType::FunctionName(name) => {
-                    vec![req_tools
-                        .iter()
-                        .find(|tool| tool.function.name == *name)
-                        .unwrap_or_else(|| panic!("Tool with name {} not found", name))
-                        .clone()]
-                }
-                ToolType::OneOf => req_tools.to_owned(),
-            };
-
-            // adds the error notification function for LLM feedback if required
-            let mut text_response_properties = Map::new();
-            text_response_properties.insert(
-                "error".to_string(),
-                serde_json::json!({
-                    "type": "string",
-                    "description": "The error or issue to notify"
-                }),
-            );
-            text_response_properties.insert(
-                "_name".to_string(),
-                serde_json::json!({
-                    "type": "string",
-                    "const": "notify_error"
-                }),
-            );
-
-            let functions: HashMap<String, serde_json::Value> = tools_to_use
-                .iter()
-                .map(|tool| {
-                    let func = tool.function.clone();
-
-                    // Clone the existing parameters, which are expected to be a JSON object
-                    let mut params = if let Value::Object(params) = &func.arguments {
-                        params.clone()
-                    } else {
-                        Map::new()
-                    };
-
-                    // Insert the function's description at the top level, outside of properties
-                    params.insert(
-                        "description".to_string(),
-                        Value::String(func.description.clone().unwrap_or_default()),
-                    );
-
-                    // Ensure 'properties' exists and is an object
-                    let properties = params
-                        .entry("properties".to_string())
-                        .or_insert_with(|| json!({}))
-                        .as_object_mut()
-                        .unwrap();
-
-                    // Insert the constant for the function name inside 'properties'
-                    properties.insert(
-                        "_name".to_string(),
-                        json!({
-                            "type": "string",
-                            "const": func.name.clone(),
-                            // "description": "The name of the function"
-                        }),
-                    );
-
-                    // Check if 'required' exists, and it is an array. If not, create an empty array.
-                    let required = params
-                        .entry("required".to_string())
-                        .or_insert_with(|| json!([]))
-                        .as_array_mut()
-                        .unwrap();
-
-                    // Add 'name' to the 'required' array if it is not already present
-                    if !required.iter().any(|r| r == "_name") {
-                        required.push(json!("_name"));
-                    }
-
-                    (func.name, Value::Object(params))
-                })
-                .chain([(
-                    "notify_error".to_string(),
-                    serde_json::json!({
-                        "properties": text_response_properties,
-                        "required": ["error", "_name"],
-                        "type": "object"
-                    }),
-                )])
-                .collect();
-
-            let tools = Tools {
-                functions_map: FunctionsMap { functions },
-                properties: Properties {
-                    function: tools_to_use
-                        .iter()
-                        .map(|tool| FunctionRef {
-                            ref_path: format!("#/$functions/{}", tool.function.name.clone()),
-                        })
-                        .chain(std::iter::once(FunctionRef {
-                            ref_path: "#/$functions/notify_error".to_string(),
-                        }))
-                        .collect(),
-                },
-            };
-
-            return Ok(Some(tools));
-        }
-        // Err(InferError::ToolError("No tools provided".to_string()))
-        Ok(None)
-    }
-}
-
-/// Batching logic
-/// Will be launched in a background Tokio task
-///
-/// Batches requests and sends them to the inference server
-#[allow(clippy::too_many_arguments)]
-async fn batching_task(
-    mut client: ShardedClient,
-    waiting_served_ratio: f32,
-    max_batch_prefill_tokens: u32,
-    max_batch_total_tokens: u32,
-    max_waiting_tokens: usize,
-    max_batch_size: Option<usize>,
-    queue: Queue,
-    shared: Arc<Shared>,
-    generation_health: Arc<AtomicBool>,
-) {
-    // Infinite loop
-    loop {
-        // Wait for a notification from the Infer struct
-        shared.batching_task.notified().await;
-
-        // Get the next batch from the queue
-        // This batch might be smaller than the maximum batch size if there are not enough requests
-        // waiting in the queue
-        while let Some((mut entries, batch, span)) = queue
-            .next_batch(
-                None,
-                max_batch_size,
-                max_batch_prefill_tokens,
-                max_batch_total_tokens,
-            )
-            .await
-        {
-            let mut cached_batch = prefill(&mut client, batch, &mut entries, &generation_health)
-                .instrument(span)
-                .await;
-            let mut waiting_tokens = 1;
-
-            // We loop until we do not receive any cached batch from the inference server (== until
-            // all requests have met their stopping criteria)
-            while let Some(batch) = cached_batch {
-                // Get current batch info
-                let batch_size = batch.size;
-                let batch_max_tokens = batch.max_tokens;
-                let mut batches = vec![batch];
-                metrics::gauge!("tgi_batch_current_size", batch_size as f64);
-                metrics::gauge!("tgi_batch_current_max_tokens", batch_max_tokens as f64);
-
-                let min_size = if waiting_tokens >= max_waiting_tokens {
-                    // If we didn't onboard any new requests since >= max_waiting_tokens, we try
-                    // to add a new batch even though its size might be small
-                    None
-                } else {
-                    // Minimum batch size
-                    Some((batch_size as f32 * waiting_served_ratio).floor() as usize)
-                };
-
-                let token_budget = max_batch_total_tokens.saturating_sub(batch_max_tokens);
-                let max_size = max_batch_size.map(|max_size| max_size - batch_size as usize);
-
-                // Try to get a new batch
-                if let Some((mut new_entries, new_batch, span)) = queue
-                    .next_batch(min_size, max_size, max_batch_prefill_tokens, token_budget)
-                    .await
-                {
-                    // Tracking metrics
-                    if min_size.is_some() {
-                        metrics::increment_counter!("tgi_batch_concat", "reason" => "backpressure");
-                    } else {
-                        metrics::increment_counter!("tgi_batch_concat", "reason" => "wait_exceeded");
-                    }
-
-                    entries.iter_mut().for_each(|(_, entry)| {
-                        // Create a new span to add the info that this entry is waiting
-                        // because a new batch is being computed
-                        let entry_waiting_span = info_span!(parent: &entry.span, "waiting");
-                        // Add relationships
-                        span.follows_from(&entry_waiting_span);
-                        entry_waiting_span.follows_from(&span);
-                        // Update entry
-                        entry.temp_span = Some(entry_waiting_span);
-                    });
-
-                    // Generate one token for this new batch to have the attention past in cache
-                    let new_cached_batch =
-                        prefill(&mut client, new_batch, &mut new_entries, &generation_health)
-                            .instrument(span)
-                            .await;
-                    // Reset waiting counter
-                    waiting_tokens = 1;
-                    // Extend current batch with the new batch
-                    if let Some(new_cached_batch) = new_cached_batch {
-                        entries.extend(new_entries);
-                        batches.push(new_cached_batch);
-                    }
-                }
-
-                // Create span for this batch to add context to inference calls
-                let next_batch_size = entries.len();
-                let next_batch_span =
-                    info_span!(parent: None, "batch", batch_size = next_batch_size);
-                entries.iter_mut().for_each(|(_, entry)| {
-                    // Create a new span to link the batch back to this entry
-                    let entry_batch_span = info_span!(parent: &entry.span, "infer");
-                    // Add relationships
-                    next_batch_span.follows_from(&entry_batch_span);
-                    entry_batch_span.follows_from(&next_batch_span);
-                    // Update entry
-                    entry.temp_span = Some(entry_batch_span);
-                });
-
-                cached_batch = decode(&mut client, batches, &mut entries, &generation_health)
-                    .instrument(next_batch_span)
-                    .await;
-                waiting_tokens += 1;
-            }
-            metrics::gauge!("tgi_batch_current_size", 0.0);
-            metrics::gauge!("tgi_batch_current_max_tokens", 0.0);
-        }
-    }
-}
-
-#[instrument(skip_all)]
-async fn prefill(
-    client: &mut ShardedClient,
-    batch: Batch,
-    entries: &mut IntMap<u64, Entry>,
-    generation_health: &Arc<AtomicBool>,
-) -> Option<CachedBatch> {
-    let start_time = Instant::now();
-    let batch_id = batch.id;
-    metrics::increment_counter!("tgi_batch_inference_count", "method" => "prefill");
-
-    match client.prefill(batch).await {
-        Ok((generations, next_batch, timings)) => {
-            // Update health
-            generation_health.store(true, Ordering::SeqCst);
-
-            let start_filtering_time = Instant::now();
-            // Send generated tokens and filter stopped entries
-            filter_send_generations(generations, entries);
-
-            // Filter next batch and remove requests that were stopped
-            let next_batch = filter_batch(client, next_batch, entries).await;
-
-            metrics::histogram!("tgi_batch_forward_duration", timings.forward.as_secs_f64(), "method" => "prefill");
-            metrics::histogram!("tgi_batch_decode_duration", timings.decode.as_secs_f64(), "method" => "prefill");
-            metrics::histogram!("tgi_batch_filter_duration", start_filtering_time.elapsed().as_secs_f64(), "method" => "prefill");
-            metrics::histogram!("tgi_batch_inference_duration", start_time.elapsed().as_secs_f64(), "method" => "prefill");
-            metrics::increment_counter!("tgi_batch_inference_success", "method" => "prefill");
-            next_batch
-        }
-        // If we have an error, we discard the whole batch
-        Err(err) => {
-            // Update health
-            generation_health.store(false, Ordering::SeqCst);
-            let _ = client.clear_cache(Some(batch_id)).await;
-            send_errors(err, entries);
-            metrics::increment_counter!("tgi_batch_inference_failure", "method" => "prefill");
-            None
-        }
-    }
-}
-
-#[instrument(skip_all)]
-async fn decode(
-    client: &mut ShardedClient,
-    batches: Vec<CachedBatch>,
-    entries: &mut IntMap<u64, Entry>,
-    generation_health: &Arc<AtomicBool>,
-) -> Option<CachedBatch> {
-    let start_time = Instant::now();
-    let batch_ids: Vec<u64> = batches.iter().map(|b| b.id).collect();
-    metrics::increment_counter!("tgi_batch_inference_count", "method" => "decode");
-
-    match client.decode(batches).await {
-        Ok((generations, next_batch, timings)) => {
-            // Update health
-            generation_health.store(true, Ordering::SeqCst);
-
-            let start_filtering_time = Instant::now();
-            // Send generated tokens and filter stopped entries
-            filter_send_generations(generations, entries);
-
-            // Filter next batch and remove requests that were stopped
-            let next_batch = filter_batch(client, next_batch, entries).await;
-
-            if let Some(concat_duration) = timings.concat {
-                metrics::histogram!("tgi_batch_concat_duration", concat_duration.as_secs_f64(), "method" => "decode");
-            }
-            metrics::histogram!("tgi_batch_forward_duration", timings.forward.as_secs_f64(), "method" => "decode");
-            metrics::histogram!("tgi_batch_decode_duration", timings.decode.as_secs_f64(), "method" => "decode");
-            metrics::histogram!("tgi_batch_filter_duration", start_filtering_time.elapsed().as_secs_f64(), "method" => "decode");
-            metrics::histogram!("tgi_batch_inference_duration", start_time.elapsed().as_secs_f64(), "method" => "decode");
-            metrics::increment_counter!("tgi_batch_inference_success", "method" => "decode");
-            next_batch
-        }
-        // If we have an error, we discard the whole batch
-        Err(err) => {
-            generation_health.store(false, Ordering::SeqCst);
-            for id in batch_ids {
-                let _ = client.clear_cache(Some(id)).await;
-            }
-            send_errors(err, entries);
-            metrics::increment_counter!("tgi_batch_inference_failure", "method" => "decode");
-            None
-        }
-    }
-}
-
-/// Filter a `batch` and remove all requests not present in `entries`
-#[instrument(skip_all)]
-async fn filter_batch(
-    client: &mut ShardedClient,
-    next_batch: Option<CachedBatch>,
-    entries: &IntMap<u64, Entry>,
-) -> Option<CachedBatch> {
-    let mut batch = next_batch?;
-
-    // No need to filter
-    if batch.size as usize == entries.len() {
-        return Some(batch);
-    }
-
-    let id = batch.id;
-
-    // Retain only requests that are still in entries
-    batch.request_ids.retain(|id| entries.contains_key(id));
-
-    if batch.request_ids.is_empty() {
-        // All requests have been filtered out
-        // Next batch is now empty
-        // Clear it from the Python shards cache
-        // We unwrap here as we need to panic since we cannot recover if this method fails
-        client.clear_cache(Some(id)).await.unwrap();
-        None
-    } else {
-        // Filter Python shard cache
-        // We unwrap here as we need to panic since we cannot recover if this method fails
-        client.filter_batch(id, batch.request_ids).await.unwrap()
-    }
-}
-
-/// Send one or multiple `InferStreamResponse` to Infer for all `entries`
-/// and filter entries
-#[instrument(skip_all)]
-fn filter_send_generations(generations: Vec<Generation>, entries: &mut IntMap<u64, Entry>) {
-    generations.into_iter().for_each(|generation| {
-        let id = generation.request_id;
-        // Get entry
-        // We can `expect` here as the request id should always be in the entries
-        let entry = entries
-            .get(&id)
-            .expect("ID not found in entries. This is a bug.");
-
-        // Create and enter a span to link this function back to the entry
-        let _span = info_span!(parent: entry.temp_span.as_ref().expect("batch_span is None. This is a bug."), "send_generation", generation = ?generation).entered();
-        // Send generation responses back to the infer task
-        // If the receive an error from the Flume channel, it means that the client dropped the
-        // request and we need to stop generating hence why we unwrap_or(true)
-        let stopped = send_responses(generation, entry).map_err(|err| {
-            tracing::error!("Entry response channel error.");
-            metrics::increment_counter!("tgi_request_failure", "err" => "dropped");
-            err
-        }).unwrap_or(true);
-        if stopped {
-            entries.remove(&id).expect("ID not found in entries. This is a bug.");
-        }
-    });
-}
-
-/// Send responses through the `entry` response channel
-fn send_responses(
-    generation: Generation,
-    entry: &Entry,
-) -> Result<bool, Box<SendError<Result<InferStreamResponse, InferError>>>> {
-    // Return directly if the channel is disconnected
-    if entry.response_tx.is_closed() {
-        metrics::increment_counter!("tgi_request_failure", "err" => "dropped");
-        return Ok(true);
-    }
-
-    let mut stopped = false;
-
-    if let Some(prefill_tokens) = generation.prefill_tokens {
-        // Send message
-        entry
-            .response_tx
-            .send(Ok(InferStreamResponse::Prefill(prefill_tokens)))?;
-    }
-
-    // Create last Token
-    let tokens_ = generation.tokens.expect("Non empty tokens in generation");
-    let n = tokens_.ids.len();
-    metrics::histogram!("tgi_request_skipped_tokens", (n - 1) as f64);
-    let mut iterator = tokens_
-        .ids
-        .into_iter()
-        .zip(tokens_.logprobs)
-        .zip(tokens_.texts)
-        .zip(tokens_.is_special)
-        .enumerate()
-        .peekable();
-    while let Some((i, (((id, logprob), text), special))) = iterator.next() {
-        let token = Token {
-            id,
-            text,
-            logprob,
-            special,
-        };
-        let top_tokens = if let Some(top_tokens_) = generation.top_tokens.get(i) {
-            top_tokens_
-                .ids
-                .iter()
-                .zip(top_tokens_.logprobs.iter())
-                .zip(top_tokens_.texts.iter())
-                .zip(top_tokens_.is_special.iter())
-                .map(|(((&id, &logprob), text), &special)| Token {
-                    id,
-                    text: text.to_string(),
-                    logprob,
-                    special,
-                })
-                .collect()
-        } else {
-            vec![]
-        };
-        match (&generation.generated_text, iterator.peek()) {
-            (Some(generated_text), None) => {
-                // Generation has ended
-                stopped = true;
-                // Send message
-                entry.response_tx.send(Ok(InferStreamResponse::End {
-                    token,
-                    top_tokens,
-                    generated_text: generated_text.clone(),
-                    queued: entry.queue_time,
-                    start: entry.batch_time.unwrap(),
-                }))?;
-            }
-            _ => {
-                // Send message
-                entry
-                    .response_tx
-                    .send(Ok(InferStreamResponse::Intermediate { token, top_tokens }))?;
-            }
-        }
-    }
-
-    Ok(stopped)
-}
-
-/// Send errors to Infer for all `entries`
-#[instrument(skip_all)]
-fn send_errors(error: ClientError, entries: &mut IntMap<u64, Entry>) {
-    entries.drain().for_each(|(_, entry)| {
-        // Create and enter a span to link this function back to the entry
-        let _send_error_span = info_span!(parent: entry.temp_span.as_ref().expect("batch_span is None. This is a bug."), "send_error").entered();
-        let err = InferError::GenerationError(error.to_string());
-        metrics::increment_counter!("tgi_request_failure", "err" => "generation");
-        tracing::error!("{err}");
-
-        // unwrap_or is valid here as we don't care if the receiver is gone.
-        entry
-            .response_tx
-            .send(Err(err))
-            .unwrap_or(());
-    });
-}
-
-#[derive(Debug)]
-pub(crate) enum InferStreamResponse {
-    // Optional first message
-    Prefill(Tokens),
-    // Intermediate messages
-    Intermediate {
-        token: Token,
-        top_tokens: Vec<Token>,
-    },
-    // Last message
-    End {
-        token: Token,
-        top_tokens: Vec<Token>,
-        generated_text: GeneratedText,
-        start: Instant,
-        queued: Instant,
-    },
-}
-
-#[derive(Debug)]
-pub(crate) struct InferResponse {
-    /// input_length is the input as perceived by the rust tokenizer in the
-    /// validation pathway. It is redundant with prefill.len() but prefill
-    /// has data only if the user asked for it. This will always be filled.
-    pub(crate) _input_length: u32,
-    pub(crate) prefill: Vec<PrefillToken>,
-    pub(crate) tokens: Vec<Token>,
-    pub(crate) generated_text: GeneratedText,
-    pub(crate) queued: Instant,
-    pub(crate) start: Instant,
-    pub(crate) top_tokens: Vec<Vec<Token>>,
-}
-
-#[derive(Debug, Error)]
-pub enum InferError {
-    #[error("Request failed during generation: {0}")]
-    GenerationError(String),
-    #[error("Model is overloaded")]
-    Overloaded(#[from] TryAcquireError),
-    #[error("Input validation error: {0}")]
-    ValidationError(#[from] ValidationError),
-    #[error("Incomplete generation")]
-    IncompleteGeneration,
-    #[error("Template error: {0}")]
-    TemplateError(#[from] minijinja::Error),
-    #[error("Tool error: {0}")]
-    ToolError(String),
-}
-
-impl InferError {
-    pub(crate) fn error_type(&self) -> &str {
-        match self {
-            InferError::GenerationError(_) => "generation",
-            InferError::Overloaded(_) => "overloaded",
-            InferError::ValidationError(_) => "validation",
-            InferError::IncompleteGeneration => "incomplete_generation",
-            InferError::TemplateError(_) => "template_error",
-            InferError::ToolError(_) => "tool_error",
-        }
-    }
-}
-
 // tests
 #[cfg(test)]
 mod tests {
-    use crate::infer::raise_exception;
-    use crate::{ChatTemplateInputs, TextMessage};
+    use crate::infer::chat_template::raise_exception;
+    use crate::infer::ChatTemplate;
+    use crate::{
+        ChatTemplateInputs, Message, MessageContent, TextMessage, TokenizerConfigToken, Tool,
+    };
     use minijinja::Environment;
 
     #[test]
@@ -1361,11 +517,11 @@ mod tests {
                 chat_template: "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\\n'  + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
                 input: ChatTemplateInputs {
                     messages: vec![
-                        TextMessage{
+                        TextMessage {
                             role: "system".to_string(),
                             content: "You are a friendly chatbot who always responds in the style of a pirate".to_string(),
                         },
-                        TextMessage{
+                        TextMessage {
                             role: "user".to_string(),
                             content: "How many helicopters can a human eat in one sitting?".to_string(),
                         },
@@ -1585,7 +741,7 @@ mod tests {
                 target: "<｜begin▁of▁sentence｜>You are an AI programming assistant, utilizing the Deepseek Coder model, developed by Deepseek Company, and you only answer questions related to computer science. For politically sensitive questions, security and privacy issues, and other non-computer science questions, you will refuse to answer\n### Instruction:\nHello, how are you?\n### Response:\nI'm doing great. How can I help you today?\n<|EOT|>\n### Instruction:\nI'd like to show off how chat templating works!\n",
             },
             // NOT INCLUDED
-            // - meetkai/functionary-medium-v2.2
+            // - meetkai/functionary-medium-v3.2
             // - fireworks-ai/firefunction-v1
             // https://github
             ChatTemplateTestItem {
@@ -1600,6 +756,19 @@ mod tests {
                 },
                 target: "You are a friendly chatbot who always responds in the style of a pirateYou are a friendly chatbot who always responds in the style of a pirate### Instruction: Hello, how are you?### Response: I'm doing great. How can I help you today?### Instruction: I'd like to show off how chat templating works!",
             },
+            ChatTemplateTestItem {
+                name: "google/shieldgemma-9b",
+                chat_template: "{{- bos_token }}\n{%- if messages[-1].role == 'user' %}\n    {{- \"You are a policy expert trying to help determine whether a user prompt is in violation of the defined safety policies.\\n\\n\" }}\n    {{- \"<start_of_turn>\\n\" }}\n    {{- \"Human Question: \" + messages[-1].content }}\n    {{- \"\\n<end_of_turn>\\n\\n\" }}\n    {{- \"Our safety principle is defined in the below:\\n\\n\" }}\n    {{- \"* \" + guideline + \"\\n\" }}\n    {{- \"\\n===\\n\\n\" }}\n    {{- \"Does the human question violate the above principle? Your answer must start with 'Yes' or 'No'. And then walk through step by step to be sure we answer correctly.\\n\\n\" }}\n{%- else %}\n    {{- \"You are a policy expert trying to help determine whether an AI response to a prompt is in violation of the defined safety policies.\\n\\n\" }}\n    {{- \"<start_of_turn>\\n\" }}\n    {{- \"Human Question: \" + messages[-2].content }}\n    {{- \"\\n<end_of_turn>\\n\" }}\n    {{- \"<start_of_turn>\\n\" }}\n    {{- \"Chatbot Response: \" + messages[-1].content }}\n    {{- \"\\n<end_of_turn>\\n\\n\" }}\n    {{- \"Our safety principle is defined in the below:\\n\\n\" }}\n    {{- \"* \" + guideline + \"\\n\" }}\n    {{- \"\\n===\\n\\n\" }}\n    {{- \"Does the Chatbot Response violate the above principle? Your answer must  start with 'Yes' or 'No'. And then walk through step by step to be sure we answer correctly.\\n\\n\" }}\n{%- endif %}\n\n",
+                input: ChatTemplateInputs {
+                    messages: example_chat_with_system.clone(),
+                    add_generation_prompt: false,
+                    bos_token: Some("<s>"),
+                    eos_token: Some("</s>"),
+                    guideline: Some("Do not use offensive language."),
+                    ..Default::default()
+                },
+                target: "<s>You are a policy expert trying to help determine whether a user prompt is in violation of the defined safety policies.\n\n<start_of_turn>\nHuman Question: I'd like to show off how chat templating works!\n<end_of_turn>\n\nOur safety principle is defined in the below:\n\n* Do not use offensive language.\n\n===\n\nDoes the human question violate the above principle? Your answer must start with 'Yes' or 'No'. And then walk through step by step to be sure we answer correctly.\n\n",
+            },
         ];
 
         #[allow(unused_variables)] // name is unused
@@ -1624,4 +793,116 @@ mod tests {
             assert_eq!(result, target);
         }
     }
+
+    #[test]
+    fn test_chat_template_invalid_with_guideline() {
+        let ct = ChatTemplate::new(
+            "{{- bos_token }}\n{%- if messages[-1].role == 'user' %}\n    {{- \"You are a policy expert trying to help determine whether a user prompt is in violation of the defined safety policies.\\n\\n\" }}\n    {{- \"<start_of_turn>\\n\" }}\n    {{- \"Human Question: \" + messages[-1].content }}\n    {{- \"\\n<end_of_turn>\\n\\n\" }}\n    {{- \"Our safety principle is defined in the below:\\n\\n\" }}\n    {{- \"* \" + guideline + \"\\n\" }}\n    {{- \"\\n===\\n\\n\" }}\n    {{- \"Does the human question violate the above principle? Your answer must start with 'Yes' or 'No'. And then walk through step by step to be sure we answer correctly.\\n\\n\" }}\n{%- else %}\n    {{- \"You are a policy expert trying to help determine whether an AI response to a prompt is in violation of the defined safety policies.\\n\\n\" }}\n    {{- \"<start_of_turn>\\n\" }}\n    {{- \"Human Question: \" + messages[-2].content }}\n    {{- \"\\n<end_of_turn>\\n\" }}\n    {{- \"<start_of_turn>\\n\" }}\n    {{- \"Chatbot Response: \" + messages[-1].content }}\n    {{- \"\\n<end_of_turn>\\n\\n\" }}\n    {{- \"Our safety principle is defined in the below:\\n\\n\" }}\n    {{- \"* \" + guideline + \"\\n\" }}\n    {{- \"\\n===\\n\\n\" }}\n    {{- \"Does the Chatbot Response violate the above principle? Your answer must  start with 'Yes' or 'No'. And then walk through step by step to be sure we answer correctly.\\n\\n\" }}\n{%- endif %}\n\n".to_string(),
+            Some(TokenizerConfigToken::String("<s>".to_string())),
+            Some(TokenizerConfigToken::String("</s>".to_string())),
+        );
+
+        // convert TextMessage to Message
+        let msgs: Vec<Message> = vec![
+            Message {
+                name: None,
+                role: "user".to_string(),
+                content: MessageContent::SingleText(
+                    "I'd like to show off how chat templating works!".to_string(),
+                ),
+            },
+            Message {
+                name: None,
+                role: "assistant".to_string(),
+                content: MessageContent::SingleText(
+                    "I'm doing great. How can I help you today?".to_string(),
+                ),
+            },
+            Message {
+                name: None,
+                role: "user".to_string(),
+                content: MessageContent::SingleText("Hello, how are you?".to_string()),
+            },
+        ];
+
+        let result = ct.apply(None, msgs, None);
+
+        match result {
+            Ok(_) => panic!("Should have failed since no guideline is provided"),
+            Err(e) => {
+                assert_eq!(e.to_string(), "Missing template vatiable: guideline")
+            }
+        }
+    }
+
+    #[test]
+    fn test_chat_template_with_default_tool_template() {
+        let ct = ChatTemplate::new(
+            "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token + ' ' }}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %}".to_string(),
+            Some(TokenizerConfigToken::String("<s>".to_string())),
+            Some(TokenizerConfigToken::String("</s>".to_string())),
+        );
+
+        // convert TextMessage to Message
+        let msgs: Vec<Message> = vec![
+            Message {
+                name: None,
+                role: "user".to_string(),
+                content: MessageContent::SingleText(
+                    "I'd like to show off how chat templating works!".to_string(),
+                ),
+            },
+            Message {
+                name: None,
+                role: "assistant".to_string(),
+                content: MessageContent::SingleText("Great! How can I help you today?".to_string()),
+            },
+            Message {
+                name: None,
+                role: "user".to_string(),
+                content: MessageContent::SingleText("Just testing".to_string()),
+            },
+        ];
+        let tools_string = r#"[{"type": "function","function": {"name": "get_current_weather","description": "Get the current weather","parameters": {"type": "object","properties": {"location": {"type": "string","description": "The city and state, e.g. San Francisco, CA"},"format": {"type": "string","enum": ["celsius", "fahrenheit"],"description": "The temperature unit to use. Infer this from the users location."}},"required": ["location", "format"]}}}]"#.to_string();
+        let tools: Vec<Tool> = serde_json::from_str(&tools_string).unwrap();
+        let tool_prompt = "This default prompt will be used".to_string();
+        let tools_and_prompt = Some((tools, tool_prompt));
+        let result = ct.apply(None, msgs, tools_and_prompt);
+        let expected = "<s>[INST] I'd like to show off how chat templating works! [/INST]Great! How can I help you today?</s> [INST] Just testing\n---\n[{\"type\":\"function\",\"function\":{\"description\":\"Get the current weather\",\"name\":\"get_current_weather\",\"arguments\":{\"properties\":{\"format\":{\"description\":\"The temperature unit to use. Infer this from the users location.\",\"enum\":[\"celsius\",\"fahrenheit\"],\"type\":\"string\"},\"location\":{\"description\":\"The city and state, e.g. San Francisco, CA\",\"type\":\"string\"}},\"required\":[\"location\",\"format\"],\"type\":\"object\"}}}]\nThis default prompt will be used [/INST]".to_string();
+        assert_eq!(result.unwrap(), expected);
+    }
+
+    #[test]
+    fn test_chat_template_with_custom_tool_template() {
+        // chat template from meta-llama/Meta-Llama-3.1-8B-Instruct
+        let ct = ChatTemplate::new(
+            "{{- bos_token }}\n{%- if not tools_in_user_message is defined %}\n    {%- set tools_in_user_message = true %}\n{%- endif %}\n{%- if not date_string is defined %}\n    {%- set date_string = \"26 Jul 2024\" %}\n{%- endif %}\n{%- if not tools is defined %}\n    {%- set tools = none %}\n{%- endif %}\n\n{#- This block extracts the system message, so we can slot it into the right place. #}\n{%- if messages[0]['role'] == 'system' %}\n    {%- set system_message = messages[0]['content']|trim %}\n    {%- set messages = messages[1:] %}\n{%- else %}\n    {%- set system_message = \"\" %}\n{%- endif %}\n\n{#- System message + builtin tools #}\n{{- \"<|start_header_id|>system<|end_header_id|>\\n\\n\" }}\n{%- if builtin_tools is defined or tools is not none %}\n    {{- \"Environment: ipython\\n\" }}\n{%- endif %}\n{%- if builtin_tools is defined %}\n    {{- \"Tools: \" + builtin_tools | reject('equalto', 'code_interpreter') | join(\", \") + \"\\n\\n\"}}\n{%- endif %}\n{{- \"Cutting Knowledge Date: December 2023\\n\" }}\n{{- \"Today Date: \" + date_string + \"\\n\\n\" }}\n{%- if tools is not none and not tools_in_user_message %}\n    {{- \"You have access to the following functions. To call a function, please respond with JSON for a function call.\" }}\n    {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n    {{- \"Do not use variables.\\n\\n\" }}\n    {%- for t in tools %}\n        {{- t | tojson(indent=4) }}\n        {{- \"\\n\\n\" }}\n    {%- endfor %}\n{%- endif %}\n{{- system_message }}\n{{- \"<|eot_id|>\" }}\n\n{#- Custom tools are passed in a user message with some extra guidance #}\n{%- if tools_in_user_message and not tools is none %}\n    {#- Extract the first user message so we can plug it in here #}\n    {%- if messages | length != 0 %}\n        {%- set first_user_message = messages[0]['content']|trim %}\n        {%- set messages = messages[1:] %}\n    {%- else %}\n        {{- raise_exception(\"Cannot put tools in the first user message when there's no first user message!\") }}\n{%- endif %}\n    {{- '<|start_header_id|>user<|end_header_id|>\\n\\n' -}}\n    {{- \"Given the following functions, please respond with a JSON for a function call \" }}\n    {{- \"with its proper arguments that best answers the given prompt.\\n\\n\" }}\n    {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n    {{- \"Do not use variables.\\n\\n\" }}\n    {%- for t in tools %}\n        {{- t | tojson(indent=4) }}\n        {{- \"\\n\\n\" }}\n    {%- endfor %}\n    {{- first_user_message + \"<|eot_id|>\"}}\n{%- endif %}\n\n{%- for message in messages %}\n    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}\n        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\\n\\n'+ message['content'] | trim + '<|eot_id|>' }}\n    {%- elif 'tool_calls' in message %}\n        {%- if not message.tool_calls|length == 1 %}\n            {{- raise_exception(\"This model only supports single tool-calls at once!\") }}\n        {%- endif %}\n        {%- set tool_call = message.tool_calls[0].function %}\n        {%- if builtin_tools is defined and tool_call.name in builtin_tools %}\n            {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n            {{- \"<|python_tag|>\" + tool_call.name + \".call(\" }}\n            {%- for arg_name, arg_val in tool_call.arguments | items %}\n                {{- arg_name + '=\"' + arg_val + '\"' }}\n                {%- if not loop.last %}\n                    {{- \", \" }}\n                {%- endif %}\n                {%- endfor %}\n            {{- \")\" }}\n        {%- else  %}\n            {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n            {{- '{\"name\": \"' + tool_call.name + '\", ' }}\n            {{- '\"parameters\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- \"}\" }}\n        {%- endif %}\n        {%- if builtin_tools is defined %}\n            {#- This means we're in ipython mode #}\n            {{- \"<|eom_id|>\" }}\n        {%- else %}\n            {{- \"<|eot_id|>\" }}\n        {%- endif %}\n    {%- elif message.role == \"tool\" or message.role == \"ipython\" %}\n        {{- \"<|start_header_id|>ipython<|end_header_id|>\\n\\n\" }}\n        {%- if message.content is mapping or message.content is iterable %}\n            {{- message.content | tojson }}\n        {%- else %}\n            {{- message.content }}\n        {%- endif %}\n        {{- \"<|eot_id|>\" }}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' }}\n{%- endif %}\n".to_string(),
+            Some(TokenizerConfigToken::String("<s>".to_string())),
+            Some(TokenizerConfigToken::String("</s>".to_string())),
+        );
+        let msgs: Vec<Message> = vec![
+            Message {
+                name: None,
+                role: "system".to_string(),
+                content: MessageContent::SingleText(
+                    "Youre a helpful assistant! Answer the users question best you can."
+                        .to_string(),
+                ),
+            },
+            Message {
+                name: None,
+                role: "user".to_string(),
+                content: MessageContent::SingleText(
+                    "What is the weather like in Brooklyn, New York?".to_string(),
+                ),
+            },
+        ];
+        let tools_string = r#"[{"type": "function","function": {"name": "get_current_weather","description": "Get the current weather","parameters": {"type": "object","properties": {"location": {"type": "string","description": "The city and state, e.g. San Francisco, CA"},"format": {"type": "string","enum": ["celsius", "fahrenheit"],"description": "The temperature unit to use. Infer this from the users location."}},"required": ["location", "format"]}}}]"#.to_string();
+        let tools: Vec<Tool> = serde_json::from_str(&tools_string).unwrap();
+        let tool_prompt = "This default prompt will be used".to_string();
+        let tools_and_prompt = Some((tools, tool_prompt));
+        let result = ct.apply(None, msgs, tools_and_prompt);
+        let expected = "<s><|start_header_id|>system<|end_header_id|>\n\nEnvironment: ipython\nCutting Knowledge Date: December 2023\nToday Date: 26 Jul 2024\n\nYoure a helpful assistant! Answer the users question best you can.<|eot_id|><|start_header_id|>user<|end_header_id|>\n\nGiven the following functions, please respond with a JSON for a function call with its proper arguments that best answers the given prompt.\n\nRespond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.Do not use variables.\n\n{\n    \"function\": {\n        \"arguments\": {\n            \"properties\": {\n                \"format\": {\n                    \"description\": \"The temperature unit to use. Infer this from the users location.\",\n                    \"enum\": [\n                        \"celsius\",\n                        \"fahrenheit\"\n                    ],\n                    \"type\": \"string\"\n                },\n                \"location\": {\n                    \"description\": \"The city and state, e.g. San Francisco, CA\",\n                    \"type\": \"string\"\n                }\n            },\n            \"required\": [\n                \"location\",\n                \"format\"\n            ],\n            \"type\": \"object\"\n        },\n        \"description\": \"Get the current weather\",\n        \"name\": \"get_current_weather\"\n    },\n    \"type\": \"function\"\n}\n\nWhat is the weather like in Brooklyn, New York?\n---\nThis default prompt will be used<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n".to_string();
+        assert_eq!(result.unwrap(), expected);
+    }
 }
diff --git a/router/src/infer/mod.rs b/router/src/infer/mod.rs
new file mode 100644
index 00000000..1c9d5620
--- /dev/null
+++ b/router/src/infer/mod.rs
@@ -0,0 +1,373 @@
+// pub(crate) mod v2;
+mod chat_template;
+pub mod tool_grammar;
+
+use crate::validation::{ValidGenerateRequest, Validation, ValidationError};
+use crate::Tool;
+use crate::{
+    ChatTemplateVersions, FinishReason, GenerateRequest, HubProcessorConfig, HubTokenizerConfig,
+    Message, PrefillToken, Token,
+};
+use async_stream::stream;
+use async_trait::async_trait;
+use chat_template::ChatTemplate;
+use futures::future::try_join_all;
+use futures::Stream;
+use minijinja::ErrorKind;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::Arc;
+use thiserror::Error;
+use tokio::sync::{OwnedSemaphorePermit, Semaphore, TryAcquireError};
+use tokio::time::Instant;
+use tokio_stream::wrappers::UnboundedReceiverStream;
+use tokio_stream::StreamExt;
+use tracing::instrument;
+
+#[async_trait]
+pub trait Backend {
+    fn schedule(
+        &self,
+        request: ValidGenerateRequest,
+    ) -> Result<UnboundedReceiverStream<Result<InferStreamResponse, InferError>>, InferError>;
+
+    async fn health(&self, current_health: bool) -> bool;
+}
+
+/// Inference struct
+#[derive(Clone)]
+pub struct Infer {
+    /// Validation
+    validation: Validation,
+    /// Request backend
+    backend: Arc<dyn Backend + Send + Sync>,
+    /// Chat template
+    chat_template: Option<ChatTemplate>,
+    /// Inference limit
+    limit_concurrent_requests: Arc<Semaphore>,
+    /// Backend health
+    backend_health: Arc<AtomicBool>,
+}
+
+impl Infer {
+    #[allow(clippy::too_many_arguments)]
+    pub(crate) fn new(
+        backend: impl Backend + Send + Sync + 'static,
+        validation: Validation,
+        max_concurrent_requests: usize,
+        tokenizer_config: HubTokenizerConfig,
+        processor_config: HubProcessorConfig,
+    ) -> Self {
+        let chat_template = tokenizer_config
+            .chat_template
+            .or(processor_config.chat_template)
+            .and_then(|t| match t {
+                ChatTemplateVersions::Single(template) => Some(template),
+                ChatTemplateVersions::Multiple(templates) => templates
+                    .into_iter()
+                    .find(|t| t.name == "default")
+                    .map(|t| t.template),
+            })
+            .map(|t| ChatTemplate::new(t, tokenizer_config.bos_token, tokenizer_config.eos_token));
+
+        // Inference limit with a semaphore
+        let semaphore = Arc::new(Semaphore::new(max_concurrent_requests));
+
+        // Backend health
+        let backend_health = Arc::new(AtomicBool::new(false));
+
+        Self {
+            validation,
+            backend: Arc::new(backend),
+            chat_template,
+            limit_concurrent_requests: semaphore,
+            backend_health,
+        }
+    }
+
+    /// Add a new request to the queue and return a stream of InferStreamResponse
+    #[instrument(skip_all)]
+    pub(crate) async fn generate_stream<'a>(
+        &'a self,
+        request: GenerateRequest,
+    ) -> Result<
+        (
+            OwnedSemaphorePermit,
+            u32, // input_length
+            impl Stream<Item = Result<InferStreamResponse, InferError>> + 'a,
+        ),
+        InferError,
+    > {
+        // Limit concurrent requests by acquiring a permit from the semaphore
+        let permit = self
+            .clone()
+            .limit_concurrent_requests
+            .try_acquire_owned()
+            .map_err(|err| {
+                metrics::counter!("tgi_request_failure", "err" => "overloaded").increment(1);
+                tracing::error!("{err}");
+                err
+            })?;
+
+        // Validate request
+        let valid_request = self.validation.validate(request).await.map_err(|err| {
+            metrics::counter!("tgi_request_failure", "err" => "validation").increment(1);
+            tracing::error!("{err}");
+            err
+        })?;
+
+        let input_length = valid_request.input_length;
+        let mut generation_stream = self.backend.schedule(valid_request)?;
+
+        // Wrap generation stream to update the backend health if the stream contains an error
+        let final_stream = stream! {
+            while let Some(response) = generation_stream.next().await {
+                yield response.inspect_err(|_err| {
+                    self.backend_health.store(false, Ordering::SeqCst);
+                })
+            }
+        };
+
+        Ok((permit, input_length, final_stream))
+    }
+
+    /// Tokenizer the input
+    #[instrument(skip_all)]
+    pub(crate) async fn tokenize(
+        &self,
+        request: GenerateRequest,
+    ) -> Result<Option<tokenizers::Encoding>, InferError> {
+        // Tokenize request
+        let inputs = request.inputs;
+        let add_special_tokens = request.add_special_tokens;
+        let truncate = request.parameters.truncate;
+        let encoding = self
+            .validation
+            .tokenize(inputs, add_special_tokens, truncate)
+            .await
+            .map_err(|err| {
+                tracing::error!("Tokenization {err}");
+                err
+            })?;
+
+        // Return Encoding
+        Ok(encoding.map(|(encoding, _)| encoding))
+    }
+
+    /// Apply the chat template to the chat request
+    #[instrument(skip_all)]
+    pub(crate) fn apply_chat_template(
+        &self,
+        guideline: Option<String>,
+        messages: Vec<Message>,
+        tools_and_prompt: Option<(Vec<Tool>, String)>,
+    ) -> Result<String, InferError> {
+        self.chat_template
+            .as_ref()
+            .ok_or_else(|| InferError::TemplateError(ErrorKind::TemplateNotFound.into()))?
+            .apply(guideline.as_deref(), messages, tools_and_prompt)
+            .map_err(|e| {
+                metrics::counter!("tgi_request_failure", "err" => "template").increment(1);
+                tracing::error!("{e}");
+                e
+            })
+    }
+
+    /// Add a new request to the queue and return a InferResponse
+    #[instrument(skip_all)]
+    pub(crate) async fn generate(
+        &self,
+        request: GenerateRequest,
+    ) -> Result<InferResponse, InferError> {
+        let use_top_tokens = request.parameters.top_n_tokens.is_some_and(|x| x > 0);
+
+        // Create stream and keep semaphore permit as long as generate lives
+        let (_permit, _input_length, stream) = self.generate_stream(request).await?;
+
+        // Return values
+        let mut result_prefill = Vec::new();
+        let mut result_tokens = Vec::new();
+        let mut result_top_tokens = Vec::new();
+        let mut result_generated_text = None;
+        let mut result_start = None;
+        let mut result_queued = None;
+
+        let mut stream = Box::pin(stream);
+
+        // Iterate on stream
+        while let Some(response) = stream.next().await {
+            match response? {
+                // Add prefill tokens
+                InferStreamResponse::Prefill(prefill_tokens) => {
+                    result_prefill = prefill_tokens;
+                }
+                // Push last token
+                InferStreamResponse::Intermediate { token, top_tokens } => {
+                    result_tokens.push(token);
+                    result_top_tokens.push(top_tokens);
+                }
+                // Final message
+                // Set return values
+                InferStreamResponse::End {
+                    token,
+                    generated_text,
+                    start,
+                    queued,
+                    top_tokens,
+                } => {
+                    result_tokens.push(token);
+                    result_top_tokens.push(top_tokens);
+                    result_generated_text = Some(generated_text);
+                    result_start = Some(start);
+                    result_queued = Some(queued)
+                }
+            }
+        }
+
+        // Check that we received a `InferStreamResponse::End` message
+        if let (Some(generated_text), Some(queued), Some(start)) =
+            (result_generated_text, result_queued, result_start)
+        {
+            Ok(InferResponse {
+                prefill: result_prefill,
+                _input_length,
+                tokens: result_tokens,
+                generated_text,
+                queued,
+                start,
+                top_tokens: if use_top_tokens {
+                    result_top_tokens
+                } else {
+                    Vec::new()
+                },
+            })
+        } else {
+            let err = InferError::IncompleteGeneration;
+            metrics::counter!("tgi_request_failure", "err" => "incomplete").increment(1);
+            tracing::error!("{err}");
+            Err(err)
+        }
+    }
+    /// Add best_of new requests to the queue and return a InferResponse of the sequence with
+    /// the highest log probability per token
+    #[instrument(skip(self, request))]
+    pub(crate) async fn generate_best_of(
+        &self,
+        request: GenerateRequest,
+        best_of: usize,
+    ) -> Result<(InferResponse, Vec<InferResponse>), InferError> {
+        // validate  best_of parameter separately
+        let best_of = self.validation.validate_best_of(best_of)?;
+
+        // create multiple generate requests
+        let mut infer_responses: Vec<InferResponse> =
+            try_join_all((0..best_of).map(|_| self.generate(request.clone()))).await?;
+
+        // get the sequence with the highest log probability per token
+        let mut max_index = 0;
+        let mut max_logprob: f32 = f32::MIN;
+
+        for (i, response) in infer_responses.iter().enumerate() {
+            // mean logprobs of the generated tokens
+            let sequence_logprob = response
+                .tokens
+                .iter()
+                .map(|token| token.logprob)
+                .sum::<f32>()
+                / response.tokens.len() as f32;
+
+            // set best sequence
+            if sequence_logprob > max_logprob {
+                max_index = i;
+                max_logprob = sequence_logprob;
+            }
+        }
+        let best_response = infer_responses.remove(max_index);
+        Ok((best_response, infer_responses))
+    }
+
+    #[instrument(skip(self))]
+    pub(crate) async fn health(&self) -> bool {
+        let health = self
+            .backend
+            .health(self.backend_health.load(Ordering::SeqCst))
+            .await;
+        self.backend_health.store(health, Ordering::SeqCst);
+        health
+    }
+}
+
+#[derive(Debug)]
+pub struct GeneratedText {
+    pub text: String,
+    pub generated_tokens: u32,
+    pub finish_reason: FinishReason,
+    pub seed: Option<u64>,
+}
+
+#[derive(Debug)]
+pub enum InferStreamResponse {
+    // Optional first message
+    Prefill(Vec<PrefillToken>),
+    // Intermediate messages
+    Intermediate {
+        token: Token,
+        top_tokens: Vec<Token>,
+    },
+    // Last message
+    End {
+        token: Token,
+        top_tokens: Vec<Token>,
+        generated_text: GeneratedText,
+        start: Instant,
+        queued: Instant,
+    },
+}
+
+#[derive(Debug)]
+pub(crate) struct InferResponse {
+    /// input_length is the input as perceived by the rust tokenizer in the
+    /// validation pathway. It is redundant with prefill.len() but prefill
+    /// has data only if the user asked for it. This will always be filled.
+    pub(crate) _input_length: u32,
+    pub(crate) prefill: Vec<PrefillToken>,
+    pub(crate) tokens: Vec<Token>,
+    pub(crate) generated_text: GeneratedText,
+    pub(crate) queued: Instant,
+    pub(crate) start: Instant,
+    pub(crate) top_tokens: Vec<Vec<Token>>,
+}
+
+#[derive(Debug, Error)]
+pub enum InferError {
+    #[error("Request failed during generation: {0}")]
+    GenerationError(String),
+    #[error("Model is overloaded")]
+    Overloaded(#[from] TryAcquireError),
+    #[error("Input validation error: {0}")]
+    ValidationError(#[from] ValidationError),
+    #[error("Incomplete generation")]
+    IncompleteGeneration,
+    #[error("Incomplete generation stream")]
+    IncompleteGenerationStream,
+    #[error("Template error: {0}")]
+    TemplateError(#[from] minijinja::Error),
+    #[error("Missing template vatiable: {0}")]
+    MissingTemplateVariable(String),
+    #[error("Tool error: {0}")]
+    ToolError(String),
+}
+
+impl InferError {
+    pub(crate) fn error_type(&self) -> &str {
+        match self {
+            InferError::GenerationError(_) => "generation",
+            InferError::Overloaded(_) => "overloaded",
+            InferError::ValidationError(_) => "validation",
+            InferError::IncompleteGeneration => "incomplete_generation",
+            InferError::IncompleteGenerationStream => "incomplete_generation_stream",
+            InferError::TemplateError(_) => "template_error",
+            InferError::MissingTemplateVariable(_) => "missing_template_variable",
+            InferError::ToolError(_) => "tool_error",
+        }
+    }
+}
diff --git a/router/src/infer/tool_grammar.rs b/router/src/infer/tool_grammar.rs
new file mode 100644
index 00000000..4fe15720
--- /dev/null
+++ b/router/src/infer/tool_grammar.rs
@@ -0,0 +1,126 @@
+use crate::infer::InferError;
+use crate::{
+    FunctionDefinition, FunctionRef, FunctionsMap, JsonSchemaTool, Properties, Tool, ToolChoice,
+    ToolType,
+};
+use serde_json::{json, Map, Value};
+use std::collections::HashMap;
+
+pub(crate) struct ToolGrammar {}
+
+impl ToolGrammar {
+    // find a tool by name
+    fn find_tool_by_name(tools: &[Tool], name: &str) -> Result<Tool, InferError> {
+        tools
+            .iter()
+            .find(|tool| tool.function.name == name)
+            .cloned()
+            .ok_or_else(|| InferError::ToolError(format!("Tool with name {} not found", name)))
+    }
+
+    pub fn apply(
+        tools: Vec<Tool>,
+        tool_choice: ToolChoice,
+    ) -> Result<(Vec<Tool>, Option<JsonSchemaTool>), InferError> {
+        // if no tools are provided, we return None
+        if tools.is_empty() {
+            return Ok((tools, None));
+        }
+
+        let tool_choice = tool_choice.0.unwrap_or(ToolType::OneOf);
+
+        let mut tools = tools.clone();
+
+        // add the notify_error function to the tools
+        let notify_error = Tool {
+            r#type: "function".to_string(),
+            function: FunctionDefinition {
+                name: "notify_error".to_string(),
+                description: Some("Notify an error or issue".to_string()),
+                arguments: json!({
+                    "type": "object",
+                    "properties": {
+                        "error": {
+                            "type": "string",
+                            "description": "The error or issue to notify"
+                        }
+                    },
+                    "required": ["error"]
+                }),
+            },
+        };
+        tools.push(notify_error);
+
+        // if tools are provided and no tool_choice we default to the OneOf
+        let tools_to_use = match tool_choice {
+            ToolType::FunctionName(name) => {
+                vec![Self::find_tool_by_name(&tools, &name)?]
+            }
+            ToolType::Function { function } => {
+                vec![Self::find_tool_by_name(&tools, &function.name)?]
+            }
+            ToolType::OneOf => tools.clone(),
+            ToolType::NoTool => return Ok((tools, None)),
+        };
+
+        let functions: HashMap<String, serde_json::Value> = tools_to_use
+            .iter()
+            .map(|tool| {
+                let func = tool.function.clone();
+
+                let mut params = Map::new();
+
+                params.insert(
+                    "description".to_string(),
+                    Value::String(func.description.unwrap_or_default()),
+                );
+
+                let mut properties = Map::new();
+                let mut required = vec![Value::String("_name".to_string())];
+
+                properties.insert(
+                    "_name".to_string(),
+                    json!({
+                        "type": "string",
+                        "const": func.name.clone(),
+                    }),
+                );
+
+                if let Value::Object(args) = func.arguments {
+                    if let Some(Value::Object(props)) = args.get("properties") {
+                        properties.extend(props.clone());
+                    }
+                    if let Some(Value::Array(reqs)) = args.get("required") {
+                        required.extend(reqs.clone());
+                    }
+                    params.insert(
+                        "additionalProperties".to_string(),
+                        Value::Bool(
+                            args.get("additionalProperties").and_then(|v| v.as_str())
+                                == Some("true"),
+                        ),
+                    );
+                }
+
+                params.insert("properties".to_string(), Value::Object(properties));
+                params.insert("required".to_string(), Value::Array(required));
+
+                (func.name, Value::Object(params))
+            })
+            .collect();
+
+        let tool_schema = JsonSchemaTool {
+            functions_map: FunctionsMap { functions },
+            properties: Properties {
+                function: tools_to_use
+                    .iter()
+                    .map(|tool| FunctionRef {
+                        ref_path: format!("#/$functions/{}", tool.function.name.clone()),
+                    })
+                    .collect(),
+            },
+        };
+
+        Ok((tools, Some(tool_schema)))
+    }
+}
diff --git a/router/src/kserve.rs b/router/src/kserve.rs
new file mode 100644
index 00000000..c53fa481
--- /dev/null
+++ b/router/src/kserve.rs
@@ -0,0 +1,245 @@
+use crate::infer::Infer;
+use crate::{
+    default_parameters,
+    server::{generate_internal, ComputeType},
+    Deserialize, ErrorResponse, GenerateParameters, GenerateRequest, Serialize, ToSchema,
+};
+use axum::extract::{Extension, Path};
+use axum::http::{HeaderMap, StatusCode};
+use axum::response::IntoResponse;
+use axum::Json;
+use futures::stream::FuturesUnordered;
+use futures::TryStreamExt;
+
+#[derive(Debug, Serialize, Deserialize, ToSchema)]
+pub struct OutputChunk {
+    pub name: String,
+    pub shape: Vec<usize>,
+    pub datatype: String,
+    pub data: Vec<u8>,
+}
+
+#[derive(Debug, Serialize, Deserialize, ToSchema)]
+pub struct InferenceOutput {
+    pub id: String,
+    pub outputs: Vec<OutputChunk>,
+}
+
+#[derive(Debug, Deserialize, ToSchema)]
+pub(crate) struct InferenceRequest {
+    pub id: String,
+    #[serde(default = "default_parameters")]
+    pub parameters: GenerateParameters,
+    pub inputs: Vec<Input>,
+    pub outputs: Vec<Output>,
+}
+
+#[derive(Debug, Serialize, Deserialize, ToSchema)]
+pub(crate) struct Input {
+    pub name: String,
+    pub shape: Vec<usize>,
+    pub datatype: String,
+    pub data: Vec<u8>,
+}
+
+#[derive(Debug, Serialize, Deserialize, ToSchema)]
+pub(crate) struct Output {
+    pub name: String,
+}
+
+#[derive(Debug, Serialize, Deserialize, ToSchema)]
+pub struct LiveResponse {
+    pub live: bool,
+}
+
+#[derive(Debug, Serialize, Deserialize, ToSchema)]
+pub struct ReadyResponse {
+    pub live: bool,
+}
+
+#[derive(Debug, Serialize, Deserialize, ToSchema)]
+pub struct MetadataServerResponse {
+    pub name: String,
+    pub version: String,
+    pub extensions: Vec<String>,
+}
+
+#[utoipa::path(
+    post,
+    tag = "Text Generation Inference",
+    path = "/v2/health/live",
+    responses(
+        (status = 200, description = "Service is live", body = LiveReponse),
+        (status = 404, description = "Service not found", body = ErrorResponse,
+            example = json!({"error": "No response"}))
+    )
+)]
+pub async fn kserve_health_live() -> Json<LiveResponse> {
+    let data = LiveResponse { live: true };
+    Json(data)
+}
+
+#[utoipa::path(
+    get,
+    tag = "Text Generation Inference",
+    path = "/v2/health/ready",
+    responses(
+        (status = 200, description = "Service is ready", body = ReadyResponse),
+        (status = 404, description = "Service not found", body = ErrorResponse,
+            example = json!({"error": "No response"}))
+    )
+)]
+pub async fn kserve_health_ready() -> Json<ReadyResponse> {
+    let data = ReadyResponse { live: true };
+    Json(data)
+}
+
+#[utoipa::path(
+    get,
+    tag = "Text Generation Inference",
+    path = "/v2",
+    responses(
+        (status = 200, description = "Metadata retrieved", body = MetadataServerResponse),
+        (status = 404, description = "Service not found", body = ErrorResponse,
+            example = json!({"error": "No response"}))
+    )
+)]
+pub async fn kerve_server_metadata() -> Json<MetadataServerResponse> {
+    let data = MetadataServerResponse {
+        name: "text-generation-inference".to_string(),
+        version: env!("CARGO_PKG_VERSION").to_string(),
+        extensions: vec![
+            "health".to_string(),
+            "models".to_string(),
+            "metrics".to_string(),
+        ],
+    };
+    Json(data)
+}
+
+#[utoipa::path(
+    get,
+    tag = "Text Generation Inference",
+    path = "/v2/models/{model_name}/versions/{model_version}",
+    responses(
+        (status = 200, description = "Model version metadata retrieved", body = MetadataServerResponse),
+        (status = 404, description = "Model or version not found", body = ErrorResponse,
+            example = json!({"error": "No response"}))
+    )
+)]
+pub async fn kserve_model_metadata(
+    Path((model_name, model_version)): Path<(String, String)>,
+) -> Json<MetadataServerResponse> {
+    let data = MetadataServerResponse {
+        name: model_name,
+        version: model_version,
+        extensions: vec!["infer".to_string(), "ready".to_string()],
+    };
+    Json(data)
+}
+
+#[utoipa::path(
+    get,
+    tag = "Text Generation Inference",
+    path = "/v2/models/{model_name}/versions/{model_version}/ready",
+    responses(
+        (status = 200, description = "Model version is ready", body = ReadyResponse),
+        (status = 404, description = "Model or version not found", body = ErrorResponse,
+            example = json!({"error": "No response"}))
+    )
+)]
+pub async fn kserve_model_metadata_ready(
+    Path((_model_name, _model_version)): Path<(String, String)>,
+) -> Json<ReadyResponse> {
+    let data = ReadyResponse { live: true };
+    Json(data)
+}
+
+#[utoipa::path(
+    post,
+    tag = "Text Generation Inference",
+    path = "/v2/models/{model_name}/versions/{model_version}/infer",
+    request_body = Json<InferenceRequest>,
+    responses(
+        (status = 200, description = "Inference executed successfully", body = InferenceOutput),
+        (status = 404, description = "Model or version not found", body = ErrorResponse,
+            example = json!({"error": "No response"}))
+    )
+)]
+pub async fn kserve_model_infer(
+    infer: Extension<Infer>,
+    Extension(compute_type): Extension<ComputeType>,
+    Json(payload): Json<InferenceRequest>,
+) -> Result<impl IntoResponse, (StatusCode, Json<ErrorResponse>)> {
+    let id = payload.id.clone();
+    let str_inputs = payload
+        .inputs
+        .iter()
+        .map(|input| {
+            std::str::from_utf8(&input.data).map_err(|e| {
+                (
+                    StatusCode::UNPROCESSABLE_ENTITY,
+                    Json(ErrorResponse {
+                        error: e.to_string(),
+                        error_type: "utf8".to_string(),
+                    }),
+                )
+            })
+        })
+        .collect::<Result<Vec<_>, _>>()?;
+
+    if str_inputs.len() != payload.outputs.len() {
+        return Err((
+            StatusCode::UNPROCESSABLE_ENTITY,
+            Json(ErrorResponse {
+                error: "Inputs and outputs length mismatch".to_string(),
+                error_type: "length mismatch".to_string(),
+            }),
+        ));
+    }
+
+    let output_chunks = str_inputs
+        .iter()
+        .zip(&payload.outputs)
+        .map(|(str_input, output)| {
+            let generate_request = GenerateRequest {
+                inputs: str_input.to_string(),
+                parameters: payload.parameters.clone(),
+            };
+            let infer = infer.clone();
+            let compute_type = compute_type.clone();
+            let span = tracing::Span::current();
+            async move {
+                generate_internal(infer, compute_type, Json(generate_request), span)
+                    .await
+                    .map(|(_, Json(generation))| {
+                        let generation_as_bytes = generation.generated_text.as_bytes().to_vec();
+                        OutputChunk {
+                            name: output.name.clone(),
+                            shape: vec![1, generation_as_bytes.len()],
+                            datatype: "BYTES".to_string(),
+                            data: generation_as_bytes,
+                        }
+                    })
+                    .map_err(|_| {
+                        (
+                            StatusCode::INTERNAL_SERVER_ERROR,
+                            Json(ErrorResponse {
+                                error: "Incomplete generation".into(),
+                                error_type: "Incomplete generation".into(),
+                            }),
+                        )
+                    })
+            }
+        })
+        .collect::<FuturesUnordered<_>>()
+        .try_collect::<Vec<_>>()
+        .await?;
+
+    let inference_output = InferenceOutput {
+        id: id.clone(),
+        outputs: output_chunks,
+    };
+
+    Ok((HeaderMap::new(), Json(inference_output)))
+}
diff --git a/router/src/lib.rs b/router/src/lib.rs
index ba1d9acc..1f6337b3 100644
--- a/router/src/lib.rs
+++ b/router/src/lib.rs
@@ -1,44 +1,63 @@
-pub mod config;
-mod health;
 /// Text Generation Inference Webserver
-mod infer;
-mod queue;
+pub mod config;
+pub mod infer;
 pub mod server;
-mod validation;
+pub mod validation;
 
-use infer::{Infer, InferError, InferStreamResponse};
-use queue::{Entry, Queue};
+#[cfg(feature = "kserve")]
+mod kserve;
+pub mod logging;
+
+pub mod usage_stats;
+mod vertex;
+
+use crate::infer::{Infer, InferError};
+use crate::server::prepare_chat_input;
 use serde::{Deserialize, Serialize};
-use tokio::sync::OwnedSemaphorePermit;
-use tokio_stream::wrappers::UnboundedReceiverStream;
 use tracing::warn;
 use utoipa::ToSchema;
 use validation::Validation;
 
-/// Type alias for generation responses
-pub(crate) type GenerateStreamResponse = (
-    OwnedSemaphorePermit,
-    u32, // input_length
-    UnboundedReceiverStream<Result<InferStreamResponse, InferError>>,
-);
-
-#[derive(Clone, Deserialize, ToSchema)]
-pub(crate) struct VertexInstance {
-    #[schema(example = "What is Deep Learning?")]
-    pub inputs: String,
-    #[schema(nullable = true, default = "null", example = "null")]
-    pub parameters: Option<GenerateParameters>,
+#[derive(PartialEq)]
+pub enum Attention {
+    Paged,
+    FlashDecoding,
+    FlashInfer,
+    Default,
 }
 
-#[derive(Deserialize, ToSchema)]
-pub(crate) struct VertexRequest {
-    #[serde(rename = "instances")]
-    pub instances: Vec<VertexInstance>,
+impl Attention {
+    pub fn block_size(&self) -> u32 {
+        match self {
+            Attention::FlashDecoding => 256,
+            Attention::FlashInfer => 1,
+            Attention::Paged => 16,
+            Attention::Default => 16,
+        }
+    }
 }
 
-#[derive(Clone, Deserialize, ToSchema, Serialize)]
-pub(crate) struct VertexResponse {
-    pub predictions: Vec<String>,
+#[derive(Debug)]
+pub struct ParseError;
+
+impl std::fmt::Display for ParseError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "Cannot parse attention value")
+    }
+}
+impl std::error::Error for ParseError {}
+
+impl std::str::FromStr for Attention {
+    type Err = ParseError;
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s {
+            "paged" => Ok(Attention::Paged),
+            "flashdecoding" => Ok(Attention::FlashDecoding),
+            "flashinfer" => Ok(Attention::FlashInfer),
+            "default" => Ok(Attention::Default),
+            _ => Err(ParseError),
+        }
+    }
 }
 
 /// Hub type
@@ -50,37 +69,92 @@ pub struct HubModelInfo {
     pub pipeline_tag: Option<String>,
 }
 
-#[derive(Debug, Clone, Deserialize, PartialEq)]
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
 pub struct ChatTemplate {
     name: String,
     template: String,
 }
 
-#[derive(Debug, Clone, Deserialize, PartialEq)]
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
 #[serde(untagged)]
 pub enum ChatTemplateVersions {
     Single(String),
     Multiple(Vec<ChatTemplate>),
 }
 
-#[derive(Debug, Clone, Deserialize, Default)]
+use std::path::Path;
+
+#[derive(Debug, Clone, Serialize, Deserialize, Default)]
 pub struct HubTokenizerConfig {
     pub chat_template: Option<ChatTemplateVersions>,
     pub completion_template: Option<String>,
-    #[serde(deserialize_with = "token_serde::deserialize")]
-    pub bos_token: Option<String>,
-    #[serde(deserialize_with = "token_serde::deserialize")]
-    pub eos_token: Option<String>,
+    pub bos_token: Option<TokenizerConfigToken>,
+    pub eos_token: Option<TokenizerConfigToken>,
+    pub tokenizer_class: Option<String>,
+    pub add_bos_token: Option<bool>,
+    pub add_eos_token: Option<bool>,
 }
 
 impl HubTokenizerConfig {
+    pub fn from_file<P: AsRef<Path>>(filename: P) -> Option<Self> {
+        std::fs::read_to_string(filename)
+            .ok()
+            .and_then(|content| serde_json::from_str(&content).ok())
+    }
+}
+
+#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)]
+#[serde(untagged)]
+pub enum TokenizerConfigToken {
+    String(String),
+    Object { content: String },
+}
+
+impl TokenizerConfigToken {
+    pub fn as_str(&self) -> &str {
+        match self {
+            TokenizerConfigToken::String(s) => s,
+            TokenizerConfigToken::Object { content } => content,
+        }
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(tag = "processor_class")]
+pub enum HubPreprocessorConfig {
+    Idefics2Processor(Idefics2Preprocessor),
+}
+
+impl HubPreprocessorConfig {
     pub fn from_file<P: AsRef<std::path::Path>>(filename: P) -> Option<Self> {
         let content = std::fs::read_to_string(filename).ok()?;
         serde_json::from_str(&content).ok()
     }
 }
 
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct Idefics2Preprocessor {
+    #[serde(default)]
+    do_image_splitting: bool,
+}
+
+#[derive(Debug, Clone, Deserialize, Default)]
+pub struct HubProcessorConfig {
+    pub chat_template: Option<ChatTemplateVersions>,
+    pub image_seq_len: usize,
+    pub processor_class: Option<String>,
+}
+
+impl HubProcessorConfig {
+    pub fn from_file<P: AsRef<Path>>(filename: P) -> Option<Self> {
+        std::fs::read_to_string(filename)
+            .ok()
+            .and_then(|content| serde_json::from_str(&content).ok())
+    }
+}
+
 #[derive(Clone, Debug, Deserialize, ToSchema, Serialize)]
+#[cfg_attr(test, derive(PartialEq))]
 #[serde(tag = "type", content = "value")]
 pub(crate) enum GrammarType {
     /// A string that represents a [JSON Schema](https://json-schema.org/).
@@ -88,41 +162,13 @@ pub(crate) enum GrammarType {
     /// JSON Schema is a declarative language that allows to annotate JSON documents
     /// with types and descriptions.
     #[serde(rename = "json")]
+    #[serde(alias = "json_object")]
     #[schema(example = json ! ({"properties": {"location":{"type": "string"}}}))]
     Json(serde_json::Value),
     #[serde(rename = "regex")]
     Regex(String),
 }
 
-mod token_serde {
-    use super::*;
-    use serde::de;
-    use serde::Deserializer;
-    use serde_json::Value;
-
-    pub fn deserialize<'de, D>(deserializer: D) -> Result<Option<String>, D::Error>
-    where
-        D: Deserializer<'de>,
-    {
-        let value = Value::deserialize(deserializer)?;
-
-        match value {
-            Value::String(s) => Ok(Some(s)),
-            Value::Object(map) => {
-                if let Some(content) = map.get("content").and_then(|v| v.as_str()) {
-                    Ok(Some(content.to_string()))
-                } else {
-                    Err(de::Error::custom(
-                        "content key not found in structured token",
-                    ))
-                }
-            }
-            Value::Null => Ok(None),
-            _ => Err(de::Error::custom("invalid token format")),
-        }
-    }
-}
-
 #[derive(Clone, Debug, Serialize, ToSchema)]
 pub struct Info {
     /// Model info
@@ -130,12 +176,13 @@ pub struct Info {
     pub model_id: String,
     #[schema(nullable = true, example = "e985a63cdc139290c5f700ff1929f0b5942cced2")]
     pub model_sha: Option<String>,
-    #[schema(example = "torch.float16")]
-    pub model_dtype: String,
-    #[schema(example = "cuda")]
-    pub model_device_type: String,
+    // #[schema(example = "torch.float16")]
+    // pub model_dtype: String,
+    // #[schema(example = "cuda")]
+    // pub model_device_type: String,
     #[schema(nullable = true, example = "text-generation")]
     pub model_pipeline_tag: Option<String>,
+
     /// Router Parameters
     #[schema(example = "128")]
     pub max_concurrent_requests: usize,
@@ -144,21 +191,14 @@ pub struct Info {
     #[schema(example = "4")]
     pub max_stop_sequences: usize,
     #[schema(example = "1024")]
-    pub max_input_length: usize,
+    pub max_input_tokens: usize,
     #[schema(example = "2048")]
     pub max_total_tokens: usize,
-    #[schema(example = "1.2")]
-    pub waiting_served_ratio: f32,
-    #[schema(example = "32000")]
-    pub max_batch_total_tokens: u32,
-    #[schema(example = "20")]
-    pub max_waiting_tokens: usize,
-    #[schema(nullable = true, example = "null")]
-    pub max_batch_size: Option<usize>,
     #[schema(example = "2")]
     pub validation_workers: usize,
     #[schema(example = "32")]
     pub max_client_batch_size: usize,
+
     /// Router Info
     #[schema(example = "text-generation-router")]
     pub router: &'static str,
@@ -171,6 +211,7 @@ pub struct Info {
 }
 
 #[derive(Clone, Debug, Deserialize, ToSchema, Default)]
+#[cfg_attr(test, derive(PartialEq))]
 pub(crate) struct GenerateParameters {
     /// Generate best_of sequences and return the one if the highest token logprobs.
     #[serde(default)]
@@ -297,6 +338,11 @@ pub(crate) struct GenerateParameters {
     #[serde(default)]
     #[schema(nullable = true, default = "null", example = "null")]
     pub grammar: Option<GrammarType>,
+
+    /// Lora adapter id
+    #[serde(default)]
+    #[schema(nullable = true, default = "null", example = "null")]
+    pub adapter_id: Option<String>,
 }
 
 fn default_max_new_tokens() -> Option<u32> {
@@ -323,33 +369,37 @@ fn default_parameters() -> GenerateParameters {
         seed: None,
         top_n_tokens: None,
         grammar: None,
+        adapter_id: None,
     }
 }
 
-mod prompt_serde {
-    use serde::{self, Deserialize, Deserializer};
-    use serde_json::Value;
+#[derive(Clone, Deserialize, Serialize, ToSchema, Debug)]
+#[serde(try_from = "PromptDeserializer")]
+pub struct Prompt(pub Vec<String>);
 
-    pub fn deserialize<'de, D>(deserializer: D) -> Result<Vec<String>, D::Error>
-    where
-        D: Deserializer<'de>,
-    {
-        let value = Value::deserialize(deserializer)?;
+#[derive(Deserialize)]
+#[serde(untagged)]
+enum PromptDeserializer {
+    Single(String),
+    Multiple(Vec<String>),
+}
+
+impl TryFrom<PromptDeserializer> for Prompt {
+    type Error = String;
+
+    fn try_from(value: PromptDeserializer) -> Result<Self, Self::Error> {
         match value {
-            Value::String(s) => Ok(vec![s]),
-            Value::Array(arr) if arr.is_empty() => Err(serde::de::Error::custom(
-                "Empty array detected. Do not use an empty array for the prompt.",
-            )),
-            Value::Array(arr) => arr
-                .iter()
-                .map(|v| match v {
-                    Value::String(s) => Ok(s.to_owned()),
-                    _ => Err(serde::de::Error::custom("Expected a string")),
-                })
-                .collect(),
-            _ => Err(serde::de::Error::custom(
-                "Expected a string or an array of strings",
-            )),
+            PromptDeserializer::Single(s) => Ok(Prompt(vec![s])),
+            PromptDeserializer::Multiple(v) => {
+                if v.is_empty() {
+                    Err(
+                        "Empty array detected. Do not use an empty array for the prompt."
+                            .to_string(),
+                    )
+                } else {
+                    Ok(Prompt(v))
+                }
+            }
         }
     }
 }
@@ -359,12 +409,11 @@ pub struct CompletionRequest {
     /// UNUSED
     #[schema(example = "mistralai/Mistral-7B-Instruct-v0.2")]
     /// ID of the model to use. See the model endpoint compatibility table for details on which models work with the Chat API.
-    pub model: String,
+    pub model: Option<String>,
 
     /// The prompt to generate completions for.
     #[schema(example = "What is Deep Learning?")]
-    #[serde(deserialize_with = "prompt_serde::deserialize")]
-    pub prompt: Vec<String>,
+    pub prompt: Prompt,
 
     /// The maximum number of tokens that can be generated in the chat completion.
     #[serde(default)]
@@ -409,10 +458,18 @@ pub struct CompletionRequest {
     pub stop: Option<Vec<String>>,
 }
 
+#[derive(Clone, Serialize, ToSchema)]
+#[serde(tag = "object")]
+enum Completion {
+    #[serde(rename = "text_completion")]
+    Chunk(Chunk),
+    #[serde(rename = "text_completion")]
+    Final(CompletionFinal),
+}
+
 #[derive(Clone, Deserialize, Serialize, ToSchema, Default)]
-pub(crate) struct Completion {
+pub(crate) struct CompletionFinal {
     pub id: String,
-    pub object: String,
     #[schema(example = "1706270835")]
     pub created: u64,
     #[schema(example = "mistralai/Mistral-7B-Instruct-v0.2")]
@@ -430,10 +487,18 @@ pub(crate) struct CompletionComplete {
     pub finish_reason: String,
 }
 
+#[derive(Clone, Deserialize, Serialize, ToSchema)]
+pub(crate) struct Chunk {
+    pub id: String,
+    pub created: u64,
+    pub choices: Vec<CompletionComplete>,
+    pub model: String,
+    pub system_fingerprint: String,
+}
+
 #[derive(Clone, Deserialize, Serialize, ToSchema)]
 pub(crate) struct ChatCompletion {
     pub id: String,
-    pub object: String,
     #[schema(example = "1706270835")]
     pub created: u64,
     #[schema(example = "mistralai/Mistral-7B-Instruct-v0.2")]
@@ -529,6 +594,15 @@ pub(crate) struct Usage {
     pub total_tokens: u32,
 }
 
+#[derive(Clone, Serialize, ToSchema)]
+#[serde(tag = "object")]
+enum CompletionType {
+    #[serde(rename = "chat.completion.chunk")]
+    ChatCompletionChunk(ChatCompletionChunk),
+    #[serde(rename = "chat.completion")]
+    ChatCompletion(ChatCompletion),
+}
+
 impl ChatCompletion {
     pub(crate) fn new(
         model: String,
@@ -565,7 +639,6 @@ impl ChatCompletion {
         };
         Self {
             id: String::new(),
-            object: "text_completion".into(),
             created,
             model,
             system_fingerprint,
@@ -574,7 +647,7 @@ impl ChatCompletion {
                 message,
                 logprobs: return_logprobs
                     .then(|| ChatCompletionLogprobs::from((details.tokens, details.top_tokens))),
-                finish_reason: details.finish_reason.to_string(),
+                finish_reason: details.finish_reason.format(true),
             }],
             usage: Usage {
                 prompt_tokens: details.prefill.len() as u32,
@@ -584,26 +657,16 @@ impl ChatCompletion {
         }
     }
 }
-#[derive(Clone, Deserialize, Serialize, ToSchema)]
-pub(crate) struct CompletionCompleteChunk {
-    pub id: String,
-    pub object: String,
-    pub created: u64,
-    pub choices: Vec<CompletionComplete>,
-    pub model: String,
-    pub system_fingerprint: String,
-}
-
 #[derive(Clone, Serialize, ToSchema)]
 pub(crate) struct ChatCompletionChunk {
     pub id: String,
-    pub object: String,
     #[schema(example = "1706270978")]
     pub created: u64,
     #[schema(example = "mistralai/Mistral-7B-Instruct-v0.2")]
     pub model: String,
     pub system_fingerprint: String,
     pub choices: Vec<ChatCompletionChoice>,
+    pub usage: Option<Usage>,
 }
 
 #[derive(Clone, Serialize, ToSchema)]
@@ -652,6 +715,7 @@ impl ChatCompletionChunk {
         created: u64,
         logprobs: Option<ChatCompletionLogprobs>,
         finish_reason: Option<String>,
+        usage: Option<Usage>,
     ) -> Self {
         let delta = match (delta, tool_calls) {
             (Some(delta), _) => ChatCompletionDelta::Chat(TextMessage {
@@ -677,7 +741,6 @@ impl ChatCompletionChunk {
         };
         Self {
             id: String::new(),
-            object: "text_completion".to_string(),
             created,
             model,
             system_fingerprint,
@@ -687,15 +750,17 @@ impl ChatCompletionChunk {
                 logprobs,
                 finish_reason,
             }],
+            usage,
         }
     }
 }
 
 #[derive(Clone, Deserialize, ToSchema, Serialize)]
+#[cfg_attr(test, derive(Debug, PartialEq, Default))]
 pub(crate) struct ChatRequest {
     #[schema(example = "mistralai/Mistral-7B-Instruct-v0.2")]
     /// [UNUSED] ID of the model to use. See the model endpoint compatibility table for details on which models work with the Chat API.
-    pub model: String,
+    pub model: Option<String>,
 
     /// A list of messages comprising the conversation so far.
     #[schema(example = "[{\"role\": \"user\", \"content\": \"What is Deep Learning?\"}]")]
@@ -778,69 +843,162 @@ pub(crate) struct ChatRequest {
     pub tools: Option<Vec<Tool>>,
 
     /// A prompt to be appended before the tools
-    #[serde(default = "default_tool_prompt")]
+    #[serde(default)]
     #[schema(
         nullable = true,
-        example = "\"You will be presented with a JSON schema representing a set of tools.\nIf the user request lacks of sufficient information to make a precise tool selection: Do not invent any tool's properties, instead notify with an error message.\n\nJSON Schema:\n\""
+        example = "Given the functions available, please respond with a JSON for a function call with its proper arguments that best answers the given prompt. Respond in the format {name: function name, parameters: dictionary of argument name and its value}.Do not use variables."
     )]
     pub tool_prompt: Option<String>,
 
     /// A specific tool to use. If not provided, the model will default to use any of the tools provided in the tools parameter.
     #[serde(default)]
     #[schema(nullable = true, example = "null")]
-    #[serde(deserialize_with = "deserialize_tool_choice::deserialize")]
-    pub tool_choice: Option<ToolType>,
+    pub tool_choice: ToolChoice,
+
+    /// Response format constraints for the generation.
+    ///
+    /// NOTE: A request can use `response_format` OR `tools` but not both.
+    #[serde(default)]
+    #[schema(nullable = true, default = "null", example = "null")]
+    pub response_format: Option<GrammarType>,
+
+    /// A guideline to be used in the chat_template
+    #[serde(default)]
+    #[schema(nullable = true, default = "null", example = "null")]
+    pub guideline: Option<String>,
+
+    /// Options for streaming response. Only set this when you set stream: true.
+    #[serde(default)]
+    #[schema(nullable = true, example = "null")]
+    pub stream_options: Option<StreamOptions>,
 }
 
-fn default_tool_prompt() -> Option<String> {
-    Some(
-        "\nYou will be presented with a JSON schema representing a set of tools.\nIf the user request lacks of sufficient information to make a precise tool selection: Do not invent any tool's properties, instead notify with an error message.\n\nJSON Schema:\n".to_string(),
-    )
-}
-#[derive(Clone, Deserialize, ToSchema, Serialize)]
-enum ToolType {
-    FunctionName(String),
-    OneOf,
-}
+impl ChatRequest {
+    fn try_into_generate(self, infer: &Infer) -> Result<(GenerateRequest, bool), InferError> {
+        let ChatRequest {
+            model,
+            max_tokens,
+            messages,
+            seed,
+            stop,
+            stream,
+            tools,
+            tool_choice,
+            tool_prompt,
+            temperature,
+            response_format,
+            guideline,
+            presence_penalty,
+            frequency_penalty,
+            top_p,
+            top_logprobs,
+            ..
+        } = self;
 
-/// Deserialize the tool choice from the JSON input or from the function name ("none" is allowed but mapped to None)
-mod deserialize_tool_choice {
-    use super::*;
-    use serde::de;
-    use serde::Deserializer;
-    use serde_json::Value;
+        let repetition_penalty = presence_penalty.map(|x| x + 2.0);
+        let max_new_tokens = max_tokens.or(Some(100));
+        let tool_prompt = tool_prompt
+            .filter(|s| !s.is_empty())
+            .unwrap_or_else(default_tool_prompt);
+        let stop = stop.unwrap_or_default();
+        // enable greedy only when temperature is 0
+        let (do_sample, temperature) = match temperature {
+            Some(temperature) if temperature == 0.0 => (false, None),
+            other => (true, other),
+        };
+        let (inputs, grammar, using_tools) = prepare_chat_input(
+            infer,
+            response_format,
+            tools,
+            tool_choice,
+            &tool_prompt,
+            guideline,
+            messages,
+        )?;
 
-    pub fn deserialize<'de, D>(deserializer: D) -> Result<Option<ToolType>, D::Error>
-    where
-        D: Deserializer<'de>,
-    {
-        let value = Value::deserialize(deserializer)?;
-
-        match value {
-            Value::String(s) => match s.as_str() {
-                "none" => Ok(None),
-                "auto" => Ok(Some(ToolType::OneOf)),
-                _ => Ok(Some(ToolType::FunctionName(s))),
+        Ok((
+            GenerateRequest {
+                inputs: inputs.to_string(),
+                add_special_tokens: false,
+                parameters: GenerateParameters {
+                    best_of: None,
+                    temperature,
+                    repetition_penalty,
+                    frequency_penalty,
+                    top_k: None,
+                    top_p,
+                    typical_p: None,
+                    do_sample,
+                    max_new_tokens,
+                    return_full_text: None,
+                    stop,
+                    truncate: None,
+                    watermark: false,
+                    details: true,
+                    decoder_input_details: !stream,
+                    seed,
+                    top_n_tokens: top_logprobs,
+                    grammar,
+                    adapter_id: model.filter(|m| *m != "tgi").map(String::from),
+                },
             },
-            Value::Object(map) => {
-                if let Some(content) = map
-                    .get("function")
-                    .and_then(|v| v.get("name"))
-                    .and_then(|v| v.as_str())
-                {
-                    Ok(Some(ToolType::FunctionName(content.to_string())))
-                } else {
-                    Err(de::Error::custom("function key not found in tool choice"))
-                }
-            }
-            Value::Null => Ok(Some(ToolType::OneOf)),
-            _ => Err(de::Error::custom("invalid token format")),
+            using_tools,
+        ))
+    }
+}
+
+#[derive(Clone, Deserialize, ToSchema, Serialize)]
+#[cfg_attr(test, derive(Debug, PartialEq))]
+struct StreamOptions {
+    /// If set, an additional chunk will be streamed before the data: [DONE] message. The usage field on this chunk shows the token usage statistics for the entire request, and the choices field will always be an empty array. All other chunks will also include a usage field, but with a null value.
+    #[schema(example = "true")]
+    include_usage: bool,
+}
+
+pub fn default_tool_prompt() -> String {
+    "\nGiven the functions available, please respond with a JSON for a function call with its proper arguments that best answers the given prompt. Respond in the format {name: function name, parameters: dictionary of argument name and its value}.Do not use variables.\n".to_string()
+}
+
+#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, ToSchema)]
+#[serde(untagged)]
+pub enum ToolType {
+    OneOf,
+    FunctionName(String),
+    Function { function: FunctionName },
+    NoTool,
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, ToSchema)]
+pub struct FunctionName {
+    pub name: String,
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Default, ToSchema)]
+#[serde(from = "ToolTypeDeserializer")]
+pub struct ToolChoice(pub Option<ToolType>);
+
+#[derive(Deserialize)]
+#[serde(untagged)]
+enum ToolTypeDeserializer {
+    String(String),
+    ToolType(ToolType),
+}
+
+impl From<ToolTypeDeserializer> for ToolChoice {
+    fn from(value: ToolTypeDeserializer) -> Self {
+        match value {
+            ToolTypeDeserializer::String(s) => match s.as_str() {
+                "none" => ToolChoice(Some(ToolType::NoTool)),
+                "auto" => ToolChoice(Some(ToolType::OneOf)),
+                _ => ToolChoice(Some(ToolType::FunctionName(s))),
+            },
+            ToolTypeDeserializer::ToolType(tool_type) => ToolChoice(Some(tool_type)),
         }
     }
 }
 
 #[derive(Debug, Deserialize, Serialize, ToSchema, PartialEq)]
-pub struct Tools {
+pub struct JsonSchemaTool {
     #[serde(flatten)]
     functions_map: FunctionsMap,
     properties: Properties,
@@ -884,6 +1042,7 @@ pub(crate) struct FunctionDefinition {
 }
 
 #[derive(Clone, Debug, Deserialize, Serialize, ToSchema)]
+#[cfg_attr(test, derive(PartialEq))]
 pub(crate) struct Tool {
     // The type of the tool. Currently, only 'function' is supported.
     #[schema(example = "function")]
@@ -898,8 +1057,8 @@ pub(crate) struct ChatTemplateInputs<'a> {
     bos_token: Option<&'a str>,
     eos_token: Option<&'a str>,
     add_generation_prompt: bool,
-    tools: Option<&'a str>,
-    tools_prompt: Option<&'a str>,
+    tools: Option<Vec<Tool>>,
+    guideline: Option<&'a str>,
 }
 
 #[derive(Clone, Deserialize, Serialize, ToSchema, Default, Debug, PartialEq)]
@@ -910,26 +1069,16 @@ pub(crate) struct ToolCall {
 }
 
 #[derive(Clone, Deserialize, ToSchema, Serialize, Debug, PartialEq)]
-struct Url {
+pub struct Url {
     url: String,
 }
 
-#[derive(Clone, Deserialize, ToSchema, Serialize, Debug, PartialEq)]
-struct ImageUrl {
-    image_url: Url,
-}
-
-#[derive(Clone, Deserialize, ToSchema, Serialize, Debug, PartialEq)]
-struct Text {
-    text: String,
-}
-
 #[derive(Clone, Deserialize, ToSchema, Serialize, Debug, PartialEq)]
 #[serde(tag = "type")]
 #[serde(rename_all = "snake_case")]
-enum MessageChunk {
-    Text(Text),
-    ImageUrl(ImageUrl),
+pub enum MessageChunk {
+    Text { text: String },
+    ImageUrl { image_url: Url },
 }
 
 #[derive(Clone, Deserialize, ToSchema, Serialize, Debug, PartialEq)]
@@ -937,35 +1086,33 @@ pub struct Message {
     #[schema(example = "user")]
     role: String,
     #[schema(example = "My name is David and I")]
-    #[serde(deserialize_with = "message_content_serde::deserialize")]
-    content: Vec<MessageChunk>,
+    pub content: MessageContent,
     #[serde(default, skip_serializing_if = "Option::is_none")]
     #[schema(example = "\"David\"")]
     name: Option<String>,
 }
 
-mod message_content_serde {
-    use super::*;
-    use serde::{Deserialize, Deserializer};
+#[derive(Clone, Deserialize, Serialize, ToSchema, Debug, PartialEq)]
+#[serde(untagged)]
+pub enum MessageContent {
+    SingleText(String),
+    MultipleChunks(Vec<MessageChunk>),
+}
 
-    pub fn deserialize<'de, D>(deserializer: D) -> Result<Vec<MessageChunk>, D::Error>
-    where
-        D: Deserializer<'de>,
-    {
-        #[derive(Deserialize)]
-        #[serde(untagged)]
-        enum Message {
-            Text(String),
-            Chunks(Vec<MessageChunk>),
-        }
-        let message: Message = Deserialize::deserialize(deserializer)?;
-        let chunks = match message {
-            Message::Text(text) => {
-                vec![MessageChunk::Text(Text { text })]
+// Pushing a chunk to a single text message will convert it to a multiple chunks message
+impl MessageContent {
+    pub fn push(&mut self, chunk: MessageChunk) {
+        match self {
+            MessageContent::SingleText(text) => {
+                *self = MessageContent::MultipleChunks(vec![
+                    MessageChunk::Text { text: text.clone() },
+                    chunk,
+                ]);
             }
-            Message::Chunks(s) => s,
-        };
-        Ok(chunks)
+            MessageContent::MultipleChunks(chunks) => {
+                chunks.push(chunk);
+            }
+        }
     }
 }
 
@@ -981,18 +1128,17 @@ impl From<Message> for TextMessage {
     fn from(value: Message) -> Self {
         TextMessage {
             role: value.role,
-            content: value
-                .content
-                .into_iter()
-                .map(|c| match c {
-                    MessageChunk::Text(Text { text }) => text,
-                    MessageChunk::ImageUrl(image) => {
-                        let url = image.image_url.url;
-                        format!("![]({url})")
-                    }
-                })
-                .collect::<Vec<_>>()
-                .join(""),
+            content: match value.content {
+                MessageContent::SingleText(text) => text,
+                MessageContent::MultipleChunks(chunks) => chunks
+                    .into_iter()
+                    .map(|chunk| match chunk {
+                        MessageChunk::Text { text } => text,
+                        MessageChunk::ImageUrl { image_url } => format!("![]({})", image_url.url),
+                    })
+                    .collect::<Vec<_>>()
+                    .join(""),
+            },
         }
     }
 }
@@ -1017,6 +1163,16 @@ pub(crate) struct GenerateRequest {
     pub inputs: String,
     #[serde(default = "default_parameters")]
     pub parameters: GenerateParameters,
+
+    /// This is used internally because some requests
+    /// already contain the templated input therefore
+    /// we shouldn't add the special tokens.
+    #[serde(default = "default_true", skip)]
+    pub add_special_tokens: bool,
+}
+
+fn default_true() -> bool {
+    true
 }
 
 #[derive(Clone, Debug, Deserialize, ToSchema)]
@@ -1034,6 +1190,7 @@ impl From<CompatGenerateRequest> for GenerateRequest {
     fn from(req: CompatGenerateRequest) -> Self {
         Self {
             inputs: req.inputs,
+            add_special_tokens: true,
             parameters: req.parameters,
         }
     }
@@ -1042,23 +1199,23 @@ impl From<CompatGenerateRequest> for GenerateRequest {
 #[derive(Debug, Serialize, ToSchema)]
 pub struct PrefillToken {
     #[schema(example = 0)]
-    id: u32,
+    pub id: u32,
     #[schema(example = "test")]
-    text: String,
+    pub text: String,
     #[schema(nullable = true, example = - 0.34)]
-    logprob: f32,
+    pub logprob: f32,
 }
 
 #[derive(Debug, Serialize, ToSchema, Clone)]
 pub struct Token {
     #[schema(example = 0)]
-    id: u32,
+    pub id: u32,
     #[schema(example = "test")]
-    text: String,
+    pub text: String,
     #[schema(nullable = true, example = - 0.34)]
-    logprob: f32,
+    pub logprob: f32,
     #[schema(example = "false")]
-    special: bool,
+    pub special: bool,
 }
 
 #[derive(Debug, Serialize, ToSchema)]
@@ -1073,10 +1230,10 @@ pub struct SimpleToken {
     stop: usize,
 }
 
-#[derive(Serialize, ToSchema)]
+#[derive(Debug, Serialize, ToSchema)]
 #[serde(rename_all(serialize = "snake_case"))]
 #[schema(example = "Length")]
-pub(crate) enum FinishReason {
+pub enum FinishReason {
     #[schema(rename = "length")]
     Length,
     #[serde(rename = "eos_token")]
@@ -1096,6 +1253,15 @@ impl std::fmt::Display for FinishReason {
     }
 }
 
+impl FinishReason {
+    pub fn format(&self, use_stop: bool) -> String {
+        match self {
+            FinishReason::EndOfSequenceToken if use_stop => "stop".to_string(),
+            _ => self.to_string(),
+        }
+    }
+}
+
 #[derive(Serialize, ToSchema)]
 pub(crate) struct BestOfSequence {
     #[schema(example = "test")]
@@ -1136,6 +1302,12 @@ pub(crate) struct GenerateResponse {
     pub details: Option<Details>,
 }
 
+#[derive(Serialize, ToSchema)]
+pub(crate) struct ChatTokenizeResponse {
+    pub(crate) tokenize_response: TokenizeResponse,
+    pub(crate) templated_text: String,
+}
+
 #[derive(Serialize, ToSchema)]
 #[serde(transparent)]
 pub(crate) struct TokenizeResponse(Vec<SimpleToken>);
@@ -1148,6 +1320,8 @@ pub(crate) struct StreamDetails {
     pub generated_tokens: u32,
     #[schema(nullable = true, example = 42)]
     pub seed: Option<u64>,
+    #[schema(example = 1)]
+    pub input_length: u32,
 }
 
 #[derive(Serialize, ToSchema)]
@@ -1168,6 +1342,34 @@ pub(crate) struct ErrorResponse {
     pub error_type: String,
 }
 
+#[derive(Serialize, Deserialize, ToSchema)]
+pub(crate) struct ModelInfo {
+    #[schema(example = "gpt2")]
+    pub id: String,
+    #[schema(example = "model")]
+    pub object: String,
+    #[schema(example = 1686935002)]
+    pub created: u64,
+    #[schema(example = "openai")]
+    pub owned_by: String,
+}
+
+#[derive(Serialize, Deserialize, ToSchema)]
+pub(crate) struct ModelsInfo {
+    #[schema(example = "list")]
+    pub object: String,
+    pub data: Vec<ModelInfo>,
+}
+
+impl Default for ModelsInfo {
+    fn default() -> Self {
+        ModelsInfo {
+            object: "list".to_string(),
+            data: Vec::new(),
+        }
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -1200,9 +1402,16 @@ mod tests {
         );
         assert_eq!(
             config.bos_token,
-            Some("<｜begin▁of▁sentence｜>".to_string())
+            Some(TokenizerConfigToken::String(
+                "<｜begin▁of▁sentence｜>".to_string()
+            ))
+        );
+        assert_eq!(
+            config.eos_token,
+            Some(TokenizerConfigToken::String(
+                "<｜end▁of▁sentence｜>".to_string()
+            ))
         );
-        assert_eq!(config.eos_token, Some("<｜end▁of▁sentence｜>".to_string()));
 
         // in this case we expect the tokens to be encoded as structured tokens
         // we want the content of the structured token
@@ -1235,9 +1444,16 @@ mod tests {
         );
         assert_eq!(
             config.bos_token,
-            Some("<｜begin▁of▁sentence｜>".to_string())
+            Some(TokenizerConfigToken::Object {
+                content: "<｜begin▁of▁sentence｜>".to_string()
+            })
+        );
+        assert_eq!(
+            config.eos_token,
+            Some(TokenizerConfigToken::Object {
+                content: "<｜end▁of▁sentence｜>".to_string()
+            })
         );
-        assert_eq!(config.eos_token, Some("<｜end▁of▁sentence｜>".to_string()));
     }
 
     #[test]
@@ -1255,14 +1471,41 @@ mod tests {
             request.messages[0],
             Message {
                 role: "user".to_string(),
-                content: vec![MessageChunk::Text(Text {
-                    text: "What is Deep Learning?".to_string()
-                }),],
+                content: MessageContent::SingleText("What is Deep Learning?".to_string()),
                 name: None
             }
         );
     }
 
+    #[test]
+    fn test_message_content_append() {
+        let mut content = MessageContent::SingleText("Initial text".to_string());
+        let chunk = MessageChunk::Text {
+            text: "Additional text".to_string(),
+        };
+
+        content.push(chunk);
+
+        match content {
+            MessageContent::MultipleChunks(chunks) => {
+                assert_eq!(chunks.len(), 2);
+                assert_eq!(
+                    chunks[0],
+                    MessageChunk::Text {
+                        text: "Initial text".to_string()
+                    }
+                );
+                assert_eq!(
+                    chunks[1],
+                    MessageChunk::Text {
+                        text: "Additional text".to_string()
+                    }
+                );
+            }
+            _ => panic!("Expected MultipleChunks, but got a different variant"),
+        }
+    }
+
     #[test]
     fn test_chat_request() {
         let json = json!({
@@ -1281,10 +1524,10 @@ mod tests {
             request.messages[0],
             Message{
                 role: "user".to_string(),
-                content: vec![
-                    MessageChunk::Text(Text { text: "Whats in this image?".to_string() }),
-                    MessageChunk::ImageUrl(ImageUrl { image_url: Url { url: "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/rabbit.png".to_string() } })
-                ],
+                content: MessageContent::MultipleChunks(vec![
+                    MessageChunk::Text { text: "Whats in this image?".to_string() },
+                    MessageChunk::ImageUrl { image_url: Url { url: "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/rabbit.png".to_string() }},
+                ]),
                 name: None
             }
         );
@@ -1294,15 +1537,36 @@ mod tests {
     fn text_message_convert() {
         let message = Message{
                 role: "user".to_string(),
-                content: vec![
-                    MessageChunk::Text(Text { text: "Whats in this image?".to_string() }),
-                    MessageChunk::ImageUrl(ImageUrl { image_url: Url { url: "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/rabbit.png".to_string() } })
-                ],
+                content: MessageContent::MultipleChunks(vec![
+                    MessageChunk::Text { text: "Whats in this image?".to_string() },
+                    MessageChunk::ImageUrl { image_url: Url { url: "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/rabbit.png".to_string() } }
+                ]),
                 name: None
             };
         let textmsg: TextMessage = message.into();
         assert_eq!(textmsg.content, "Whats in this image?![](https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/rabbit.png)");
     }
+
+    #[test]
+    fn test_chat_stream_options() {
+        let json = json!({
+            "model": "",
+            "stream_options": {"include_usage": true},
+            "messages": [{
+                "role": "user",
+                "content": "Hello"
+            }]
+        });
+        let request: ChatRequest = serde_json::from_str(json.to_string().as_str()).unwrap();
+
+        assert!(matches!(
+            request.stream_options,
+            Some(StreamOptions {
+                include_usage: true
+            })
+        ));
+    }
+
     #[test]
     fn openai_output() {
         let message = OutputMessage::ChatMessage(TextMessage {
diff --git a/router/src/logging.rs b/router/src/logging.rs
new file mode 100644
index 00000000..5a98ef57
--- /dev/null
+++ b/router/src/logging.rs
@@ -0,0 +1,81 @@
+use opentelemetry::sdk::propagation::TraceContextPropagator;
+use opentelemetry::sdk::trace;
+use opentelemetry::sdk::trace::Sampler;
+use opentelemetry::sdk::Resource;
+use opentelemetry::{global, KeyValue};
+use opentelemetry_otlp::WithExportConfig;
+use tracing_subscriber::layer::SubscriberExt;
+use tracing_subscriber::util::SubscriberInitExt;
+use tracing_subscriber::{filter::LevelFilter, EnvFilter, Layer};
+
+/// Init logging using env variables LOG_LEVEL and LOG_FORMAT:
+///     - otlp_endpoint is an optional URL to an Open Telemetry collector
+///     - otlp_service_name service name to appear in APM
+///     - LOG_LEVEL may be TRACE, DEBUG, INFO, WARN or ERROR (default to INFO)
+///     - LOG_FORMAT may be TEXT or JSON (default to TEXT)
+///     - LOG_COLORIZE may be "false" or "true" (default to "true" or ansi supported platforms)
+pub fn init_logging(otlp_endpoint: Option<String>, otlp_service_name: String, json_output: bool) {
+    let mut layers = Vec::new();
+
+    // STDOUT/STDERR layer
+    let ansi = std::env::var("LOG_COLORIZE") != Ok("1".to_string());
+    let fmt_layer = tracing_subscriber::fmt::layer()
+        .with_file(true)
+        .with_ansi(ansi)
+        .with_line_number(true);
+
+    let fmt_layer = match json_output {
+        true => fmt_layer.json().flatten_event(true).boxed(),
+        false => fmt_layer.boxed(),
+    };
+    layers.push(fmt_layer);
+
+    // OpenTelemetry tracing layer
+    if let Some(otlp_endpoint) = otlp_endpoint {
+        global::set_text_map_propagator(TraceContextPropagator::new());
+
+        let tracer = opentelemetry_otlp::new_pipeline()
+            .tracing()
+            .with_exporter(
+                opentelemetry_otlp::new_exporter()
+                    .tonic()
+                    .with_endpoint(otlp_endpoint),
+            )
+            .with_trace_config(
+                trace::config()
+                    .with_resource(Resource::new(vec![KeyValue::new(
+                        "service.name",
+                        otlp_service_name,
+                    )]))
+                    .with_sampler(Sampler::AlwaysOn),
+            )
+            .install_batch(opentelemetry::runtime::Tokio);
+
+        if let Ok(tracer) = tracer {
+            layers.push(tracing_opentelemetry::layer().with_tracer(tracer).boxed());
+            init_tracing_opentelemetry::init_propagator().unwrap();
+        };
+    }
+
+    // Filter events with LOG_LEVEL
+    let varname = "LOG_LEVEL";
+    let env_filter = if let Ok(log_level) = std::env::var(varname) {
+        // Override to avoid simple logs to be spammed with tokio level informations
+        let log_level = match &log_level[..] {
+            "warn" => "text_generation_launcher=warn,text_generation_router=warn",
+            "info" => "text_generation_launcher=info,text_generation_router=info",
+            "debug" => "text_generation_launcher=debug,text_generation_router=debug",
+            log_level => log_level,
+        };
+        EnvFilter::builder()
+            .with_default_directive(LevelFilter::INFO.into())
+            .parse_lossy(log_level)
+    } else {
+        EnvFilter::new("info")
+    };
+
+    tracing_subscriber::registry()
+        .with(env_filter)
+        .with(layers)
+        .init();
+}
diff --git a/router/src/main.rs b/router/src/main.rs.back
similarity index 64%
rename from router/src/main.rs
rename to router/src/main.rs.back
index 9f78a5b2..36879aa4 100644
--- a/router/src/main.rs
+++ b/router/src/main.rs.back
@@ -1,7 +1,6 @@
-/// Copyright (C) 2024 Habana Labs, Ltd. an Intel Company.
-
 use axum::http::HeaderValue;
 use clap::Parser;
+use clap::Subcommand;
 use hf_hub::api::tokio::{Api, ApiBuilder, ApiRepo};
 use hf_hub::{Cache, Repo, RepoType};
 use opentelemetry::sdk::propagation::TraceContextPropagator;
@@ -10,16 +9,17 @@ use opentelemetry::sdk::trace::Sampler;
 use opentelemetry::sdk::Resource;
 use opentelemetry::{global, KeyValue};
 use opentelemetry_otlp::WithExportConfig;
-use std::env;
 use std::fs::File;
 use std::io::BufReader;
 use std::net::{IpAddr, Ipv4Addr, SocketAddr};
 use std::path::{Path, PathBuf};
-use text_generation_client::{ClientError, ShardedClient};
 use text_generation_router::config::Config;
-use text_generation_router::{server, HubModelInfo, HubTokenizerConfig};
+use text_generation_router::usage_stats;
+use text_generation_router::{
+    server, HubModelInfo, HubPreprocessorConfig, HubProcessorConfig, HubTokenizerConfig,
+};
 use thiserror::Error;
-use tokenizers::Tokenizer;
+use tokenizers::{processors::template::TemplateProcessing, Tokenizer};
 use tower_http::cors::AllowOrigin;
 use tracing_subscriber::layer::SubscriberExt;
 use tracing_subscriber::util::SubscriberInitExt;
@@ -29,6 +29,9 @@ use tracing_subscriber::{filter::LevelFilter, EnvFilter, Layer};
 #[derive(Parser, Debug)]
 #[clap(author, version, about, long_about = None)]
 struct Args {
+    #[command(subcommand)]
+    command: Option<Commands>,
+
     #[clap(default_value = "128", long, env)]
     max_concurrent_requests: usize,
     #[clap(default_value = "2", long, env)]
@@ -69,9 +72,13 @@ struct Args {
     json_output: bool,
     #[clap(long, env)]
     otlp_endpoint: Option<String>,
+    #[clap(default_value = "text-generation-inference.router", long, env)]
+    otlp_service_name: String,
     #[clap(long, env)]
     cors_allow_origin: Option<Vec<String>>,
     #[clap(long, env)]
+    api_key: Option<String>,
+    #[clap(long, env)]
     ngrok: bool,
     #[clap(long, env)]
     ngrok_authtoken: Option<String>,
@@ -83,12 +90,21 @@ struct Args {
     disable_grammar_support: bool,
     #[clap(default_value = "4", long, env)]
     max_client_batch_size: usize,
+    #[clap(long, env, default_value_t)]
+    disable_usage_stats: bool,
+    #[clap(long, env, default_value_t)]
+    disable_crash_reports: bool,
+}
+
+#[derive(Debug, Subcommand)]
+enum Commands {
+    PrintSchema,
 }
 
 #[tokio::main]
 async fn main() -> Result<(), RouterError> {
-    // Get args
     let args = Args::parse();
+
     // Pattern match configuration
     let Args {
         max_concurrent_requests,
@@ -111,17 +127,28 @@ async fn main() -> Result<(), RouterError> {
         validation_workers,
         json_output,
         otlp_endpoint,
+        otlp_service_name,
         cors_allow_origin,
+        api_key,
         ngrok,
         ngrok_authtoken,
         ngrok_edge,
         messages_api_enabled,
         disable_grammar_support,
         max_client_batch_size,
+        disable_usage_stats,
+        disable_crash_reports,
+        command,
     } = args;
 
-    // Launch Tokio runtime
-    init_logging(otlp_endpoint, json_output);
+    let print_schema_command = match command {
+        Some(Commands::PrintSchema) => true,
+        None => {
+            // only init logging if we are not running the print schema command
+            init_logging(otlp_endpoint, otlp_service_name, json_output);
+            false
+        }
+    };
 
     // Validate args
     if max_input_tokens >= max_total_tokens {
@@ -148,25 +175,6 @@ async fn main() -> Result<(), RouterError> {
         }
     }
 
-    let (max_batch_size, max_batch_total_tokens) = match (max_batch_size, max_batch_total_tokens) {
-        (Some(_max_batch_size), Some(_max_batch_total_tokens)) => {
-            if (_max_batch_total_tokens as usize / max_total_tokens) != _max_batch_size {
-                tracing::warn!("max_batch_size was set to {_max_batch_size} while max_batch_total_tokens to {_max_batch_total_tokens}");
-                tracing::warn!("These values are not match, so max_batch_size will be preferred");
-                (Some(_max_batch_size), Some((_max_batch_size * max_total_tokens) as u32))
-            } else {
-                (Some(_max_batch_size), Some(_max_batch_total_tokens))
-            }
-        },
-        (Some(_max_batch_size), None) => (
-            Some(_max_batch_size), Some((_max_batch_size * max_total_tokens) as u32)
-        ),
-        (None, Some(_max_batch_total_tokens)) => (
-            Some(_max_batch_total_tokens as usize / max_total_tokens), Some(_max_batch_total_tokens)
-        ),
-        (None, None) => (None, None),
-    };
-
     // CORS allowed origins
     // map to go inside the option and then map to parse from String to HeaderValue
     // Finally, convert to AllowOrigin
@@ -212,7 +220,11 @@ async fn main() -> Result<(), RouterError> {
     }
     let api = if use_api {
         if std::env::var("HF_HUB_OFFLINE") == Ok("1".to_string()) {
-            let cache = Cache::default();
+            let cache = std::env::var("HUGGINGFACE_HUB_CACHE")
+                .map_err(|_| ())
+                .map(|cache_dir| Cache::new(cache_dir.into()))
+                .unwrap_or_else(|_| Cache::default());
+
             tracing::warn!("Offline mode active using cache defaults");
             Type::Cache(cache)
         } else {
@@ -230,14 +242,20 @@ async fn main() -> Result<(), RouterError> {
     };
 
     // Load tokenizer and model info
-    let skip_tokenizer_in_tgi = env::var("SKIP_TOKENIZER_IN_TGI")
-        .ok()
-        .map_or(false, |value| value.to_lowercase() == "true");
-    let (tokenizer_filename, config_filename, tokenizer_config_filename, model_info) = match api {
+    let (
+        tokenizer_filename,
+        config_filename,
+        tokenizer_config_filename,
+        preprocessor_config_filename,
+        processor_config_filename,
+        model_info,
+    ) = match api {
         Type::None => (
             Some(local_path.join("tokenizer.json")),
             Some(local_path.join("config.json")),
             Some(local_path.join("tokenizer_config.json")),
+            Some(local_path.join("preprocessor_config.json")),
+            Some(local_path.join("processor_config.json")),
             None,
         ),
         Type::Api(api) => {
@@ -253,6 +271,8 @@ async fn main() -> Result<(), RouterError> {
             };
             let config_filename = api_repo.get("config.json").await.ok();
             let tokenizer_config_filename = api_repo.get("tokenizer_config.json").await.ok();
+            let preprocessor_config_filename = api_repo.get("preprocessor_config.json").await.ok();
+            let processor_config_filename = api_repo.get("processor_config.json").await.ok();
 
             let model_info = if let Some(model_info) = get_model_info(&api_repo).await {
                 Some(model_info)
@@ -264,6 +284,8 @@ async fn main() -> Result<(), RouterError> {
                 tokenizer_filename,
                 config_filename,
                 tokenizer_config_filename,
+                preprocessor_config_filename,
+                processor_config_filename,
                 model_info,
             )
         }
@@ -277,15 +299,12 @@ async fn main() -> Result<(), RouterError> {
                 repo.get("tokenizer.json"),
                 repo.get("config.json"),
                 repo.get("tokenizer_config.json"),
+                repo.get("preprocessor_config.json"),
+                repo.get("processor_config.json"),
                 None,
             )
         }
     };
-    let tokenizer: Option<Tokenizer> = if skip_tokenizer_in_tgi {
-        None
-    } else {
-        tokenizer_filename.and_then(|filename| Tokenizer::from_file(filename).ok())
-    };
     let config: Option<Config> = config_filename.and_then(|filename| {
         std::fs::read_to_string(filename)
             .ok()
@@ -315,6 +334,28 @@ async fn main() -> Result<(), RouterError> {
         tracing::warn!("Could not find tokenizer config locally and no API specified");
         HubTokenizerConfig::default()
     });
+    let tokenizer_class = tokenizer_config.tokenizer_class.clone();
+
+    let tokenizer: Option<Tokenizer> = tokenizer_filename.and_then(|filename| {
+        let mut tokenizer = Tokenizer::from_file(filename).ok();
+        if let Some(tokenizer) = &mut tokenizer {
+            if let Some(class) = &tokenizer_config.tokenizer_class {
+                if class == "LlamaTokenizer" || class == "LlamaTokenizerFast"{
+                    if let Ok(post_processor) = create_post_processor(tokenizer, &tokenizer_config) {
+                        tracing::info!("Overriding LlamaTokenizer with TemplateProcessing to follow python override defined in https://github.com/huggingface/transformers/blob/4aa17d00690b7f82c95bb2949ea57e22c35b4336/src/transformers/models/llama/tokenization_llama_fast.py#L203-L205");
+                        tokenizer.with_post_processor(post_processor);
+                    }
+                }
+            }
+        }
+        tokenizer
+    });
+
+    let preprocessor_config =
+        preprocessor_config_filename.and_then(HubPreprocessorConfig::from_file);
+    let processor_config = processor_config_filename
+        .and_then(HubProcessorConfig::from_file)
+        .unwrap_or_default();
 
     tracing::info!("Using config {config:?}");
     if tokenizer.is_none() {
@@ -331,60 +372,6 @@ async fn main() -> Result<(), RouterError> {
         Some(pipeline_tag) => pipeline_tag.as_str() == "text-generation",
     };
 
-    // Instantiate sharded client from the master unix socket
-    let mut sharded_client = ShardedClient::connect_uds(master_shard_uds_path)
-        .await
-        .map_err(RouterError::Connection)?;
-    // Clear the cache; useful if the webserver rebooted
-    sharded_client
-        .clear_cache(None)
-        .await
-        .map_err(RouterError::Cache)?;
-    // Get info from the shard
-    let shard_info = sharded_client.info().await.map_err(RouterError::Info)?;
-
-    // Warmup model
-    tracing::info!("Warming up model");
-    let max_supported_batch_total_tokens = match sharded_client
-        .warmup(
-            max_input_tokens as u32,
-            max_batch_prefill_tokens,
-            max_total_tokens as u32,
-            max_batch_size,
-            &model_info.model_id
-        )
-        .await
-        .map_err(RouterError::Warmup)?
-    {
-        // Older models do not support automatic max-batch-total-tokens
-        None => {
-            let max_batch_total_tokens = max_batch_total_tokens
-                .unwrap_or(16000.max((max_total_tokens as u32).max(max_batch_prefill_tokens)));
-            tracing::warn!("Model does not support automatic max batch total tokens");
-            max_batch_total_tokens
-        }
-        // Flash attention models return their max supported total tokens
-        Some(max_supported_batch_total_tokens) => {
-            // Warn if user added his own max-batch-total-tokens as we will ignore it
-            if max_batch_total_tokens.is_some() {
-                tracing::warn!(
-                    "`--max-batch-total-tokens` is deprecated for Flash \
-                        Attention models."
-                );
-                tracing::warn!(
-                    "Inferred max batch total tokens: {max_supported_batch_total_tokens}"
-                );
-            }
-            if max_total_tokens as u32 > max_supported_batch_total_tokens {
-                return Err(RouterError::ArgumentValidation(format!("`max_total_tokens` must be <= `max_batch_total_tokens`. Given: {max_total_tokens} and {max_supported_batch_total_tokens}")));
-            }
-
-            max_supported_batch_total_tokens
-        }
-    };
-    tracing::info!("Setting max batch total tokens to {max_supported_batch_total_tokens}");
-    tracing::info!("Connected");
-
     // Determine the server port based on the feature and environment variable.
     let port = if cfg!(feature = "google") {
         std::env::var("AIP_HTTP_PORT")
@@ -402,10 +389,49 @@ async fn main() -> Result<(), RouterError> {
         }
     };
 
+    // Only send usage stats when TGI is run in container and the function returns Some
+    let is_container = matches!(usage_stats::is_container(), Ok(true));
+
+    let user_agent = if !disable_usage_stats && is_container {
+        let reduced_args = usage_stats::Args::new(
+            config.clone(),
+            tokenizer_class,
+            max_concurrent_requests,
+            max_best_of,
+            max_stop_sequences,
+            max_top_n_tokens,
+            max_input_tokens,
+            max_total_tokens,
+            waiting_served_ratio,
+            max_batch_prefill_tokens,
+            max_batch_total_tokens,
+            max_waiting_tokens,
+            max_batch_size,
+            revision,
+            validation_workers,
+            messages_api_enabled,
+            disable_grammar_support,
+            max_client_batch_size,
+            disable_usage_stats,
+            disable_crash_reports,
+        );
+        Some(usage_stats::UserAgent::new(reduced_args))
+    } else {
+        None
+    };
+
+    if let Some(ref ua) = user_agent {
+        let start_event =
+            usage_stats::UsageStatsEvent::new(ua.clone(), usage_stats::EventType::Start, None);
+        tokio::spawn(async move {
+            start_event.send().await;
+        });
+    };
+
     // Run server
-    server::run(
+    let result = server::run(
+        master_shard_uds_path,
         model_info,
-        shard_info,
         compat_return_full_text,
         max_concurrent_requests,
         max_best_of,
@@ -415,33 +441,70 @@ async fn main() -> Result<(), RouterError> {
         max_total_tokens,
         waiting_served_ratio,
         max_batch_prefill_tokens,
-        max_supported_batch_total_tokens,
+        max_batch_total_tokens,
         max_waiting_tokens,
         max_batch_size,
-        sharded_client,
         tokenizer,
         config,
         validation_workers,
         addr,
         cors_allow_origin,
+        api_key,
         ngrok,
         ngrok_authtoken,
         ngrok_edge,
         tokenizer_config,
+        preprocessor_config,
+        processor_config,
         messages_api_enabled,
         disable_grammar_support,
         max_client_batch_size,
+        print_schema_command,
     )
-    .await?;
-    Ok(())
+    .await;
+
+    match result {
+        Ok(_) => {
+            if let Some(ref ua) = user_agent {
+                let stop_event = usage_stats::UsageStatsEvent::new(
+                    ua.clone(),
+                    usage_stats::EventType::Stop,
+                    None,
+                );
+                stop_event.send().await;
+            };
+            Ok(())
+        }
+        Err(e) => {
+            if let Some(ref ua) = user_agent {
+                if !disable_crash_reports {
+                    let error_event = usage_stats::UsageStatsEvent::new(
+                        ua.clone(),
+                        usage_stats::EventType::Error,
+                        Some(e.to_string()),
+                    );
+                    error_event.send().await;
+                } else {
+                    let unknow_error_event = usage_stats::UsageStatsEvent::new(
+                        ua.clone(),
+                        usage_stats::EventType::Error,
+                        Some("unknow_error".to_string()),
+                    );
+                    unknow_error_event.send().await;
+                }
+            };
+            Err(RouterError::WebServer(e))
+        }
+    }
 }
 
 /// Init logging using env variables LOG_LEVEL and LOG_FORMAT:
 ///     - otlp_endpoint is an optional URL to an Open Telemetry collector
+///     - otlp_service_name service name to appear in APM
 ///     - LOG_LEVEL may be TRACE, DEBUG, INFO, WARN or ERROR (default to INFO)
 ///     - LOG_FORMAT may be TEXT or JSON (default to TEXT)
 ///     - LOG_COLORIZE may be "false" or "true" (default to "true" or ansi supported platforms)
-fn init_logging(otlp_endpoint: Option<String>, json_output: bool) {
+fn init_logging(otlp_endpoint: Option<String>, otlp_service_name: String, json_output: bool) {
     let mut layers = Vec::new();
 
     // STDOUT/STDERR layer
@@ -472,7 +535,7 @@ fn init_logging(otlp_endpoint: Option<String>, json_output: bool) {
                 trace::config()
                     .with_resource(Resource::new(vec![KeyValue::new(
                         "service.name",
-                        "text-generation-inference.router",
+                        otlp_service_name,
                     )]))
                     .with_sampler(Sampler::AlwaysOn),
             )
@@ -569,20 +632,117 @@ pub async fn get_tokenizer_config(api_repo: &ApiRepo) -> Option<HubTokenizerConf
     Some(tokenizer_config)
 }
 
+/// Create a post_processor for the LlamaTokenizer
+pub fn create_post_processor(
+    tokenizer: &Tokenizer,
+    tokenizer_config: &HubTokenizerConfig,
+) -> Result<TemplateProcessing, tokenizers::processors::template::TemplateProcessingBuilderError> {
+    let add_bos_token = tokenizer_config.add_bos_token.unwrap_or(true);
+    let add_eos_token = tokenizer_config.add_eos_token.unwrap_or(false);
+
+    let bos_token = tokenizer_config.bos_token.as_ref();
+    let eos_token = tokenizer_config.eos_token.as_ref();
+
+    if add_bos_token && bos_token.is_none() {
+        panic!("add_bos_token = true but bos_token is None");
+    }
+
+    if add_eos_token && eos_token.is_none() {
+        panic!("add_eos_token = true but eos_token is None");
+    }
+
+    let mut single = Vec::new();
+    let mut pair = Vec::new();
+    let mut special_tokens = Vec::new();
+
+    if add_bos_token {
+        if let Some(bos) = bos_token {
+            let bos_token_id = tokenizer
+                .token_to_id(bos.as_str())
+                .expect("Should have found the bos token id");
+            special_tokens.push((bos.as_str(), bos_token_id));
+            single.push(format!("{}:0", bos.as_str()));
+            pair.push(format!("{}:0", bos.as_str()));
+        }
+    }
+
+    single.push("$A:0".to_string());
+    pair.push("$A:0".to_string());
+
+    if add_eos_token {
+        if let Some(eos) = eos_token {
+            let eos_token_id = tokenizer
+                .token_to_id(eos.as_str())
+                .expect("Should have found the eos token id");
+            special_tokens.push((eos.as_str(), eos_token_id));
+            single.push(format!("{}:0", eos.as_str()));
+            pair.push(format!("{}:0", eos.as_str()));
+        }
+    }
+
+    if add_bos_token {
+        if let Some(bos) = bos_token {
+            pair.push(format!("{}:1", bos.as_str()));
+        }
+    }
+
+    pair.push("$B:1".to_string());
+
+    if add_eos_token {
+        if let Some(eos) = eos_token {
+            pair.push(format!("{}:1", eos.as_str()));
+        }
+    }
+
+    let post_processor = TemplateProcessing::builder()
+        .try_single(single)?
+        .try_pair(pair)?
+        .special_tokens(special_tokens)
+        .build()?;
+
+    Ok(post_processor)
+}
+
 #[derive(Debug, Error)]
 enum RouterError {
     #[error("Argument validation error: {0}")]
     ArgumentValidation(String),
-    #[error("Unable to connect to the Python model shards: {0}")]
-    Connection(ClientError),
-    #[error("Unable to clear the Python model shards cache: {0}")]
-    Cache(ClientError),
-    #[error("Unable to get the Python model shards info: {0}")]
-    Info(ClientError),
-    #[error("Unable to warmup the Python model shards: {0}")]
-    Warmup(ClientError),
+    #[error("WebServer error: {0}")]
+    WebServer(#[from] server::WebServerError),
     #[error("Tokio runtime failed to start: {0}")]
     Tokio(#[from] std::io::Error),
-    #[error("Axum webserver failed: {0}")]
-    Axum(#[from] axum::BoxError),
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use text_generation_router::TokenizerConfigToken;
+
+    #[test]
+    fn test_create_post_processor() {
+        let tokenizer_config = HubTokenizerConfig {
+            add_bos_token: None,
+            add_eos_token: None,
+            bos_token: Some(TokenizerConfigToken::String("<s>".to_string())),
+            eos_token: Some(TokenizerConfigToken::String("</s>".to_string())),
+            chat_template: None,
+            tokenizer_class: None,
+            completion_template: None,
+        };
+
+        let tokenizer =
+            Tokenizer::from_pretrained("TinyLlama/TinyLlama-1.1B-Chat-v1.0", None).unwrap();
+        let post_processor = create_post_processor(&tokenizer, &tokenizer_config).unwrap();
+
+        let expected = TemplateProcessing::builder()
+            .try_single("<s>:0 $A:0")
+            .unwrap()
+            .try_pair("<s>:0 $A:0 <s>:1 $B:1")
+            .unwrap()
+            .special_tokens(vec![("<s>".to_string(), 1)])
+            .build()
+            .unwrap();
+
+        assert_eq!(post_processor, expected);
+    }
 }
diff --git a/router/src/server.rs b/router/src/server.rs
index b9287080..1a909f0f 100644
--- a/router/src/server.rs
+++ b/router/src/server.rs
@@ -1,26 +1,33 @@
-/// Copyright (C) 2024 Habana Labs, Ltd. an Intel Company.
-
-use crate::config::Config;
 /// HTTP Server logic
-use crate::health::Health;
-use crate::infer::{InferError, InferResponse, InferStreamResponse, ToolGrammar};
+use crate::config::Config;
+use crate::infer::tool_grammar::ToolGrammar;
+use crate::infer::{Backend, Infer, InferError, InferResponse, InferStreamResponse};
+#[cfg(feature = "kserve")]
+use crate::kserve::{
+    kerve_server_metadata, kserve_health_live, kserve_health_ready, kserve_model_infer,
+    kserve_model_metadata, kserve_model_metadata_ready,
+};
 use crate::validation::ValidationError;
+use crate::vertex::vertex_compatibility;
+use crate::ChatTokenizeResponse;
 use crate::{
-    BestOfSequence, Details, ErrorResponse, FinishReason, GenerateParameters, GenerateRequest,
-    GenerateResponse, GrammarType, HubModelInfo, HubTokenizerConfig, Infer, Info, Message,
-    PrefillToken, SimpleToken, StreamDetails, StreamResponse, Token, TokenizeResponse, Usage,
-    Validation,
+    usage_stats, BestOfSequence, Details, ErrorResponse, FinishReason, FunctionName,
+    GenerateParameters, GenerateRequest, GenerateResponse, GrammarType, HubModelInfo,
+    HubProcessorConfig, HubTokenizerConfig, Info, Message, MessageChunk, MessageContent,
+    OutputMessage, PrefillToken, SimpleToken, StreamDetails, StreamOptions, StreamResponse,
+    TextMessage, Token, TokenizeResponse, ToolCallDelta, ToolCallMessage, Url, Usage, Validation,
 };
 use crate::{
     ChatCompletion, ChatCompletionChoice, ChatCompletionChunk, ChatCompletionComplete,
     ChatCompletionDelta, ChatCompletionLogprob, ChatCompletionLogprobs, ChatCompletionTopLogprob,
-    ChatRequest, CompatGenerateRequest, Completion, CompletionComplete, CompletionCompleteChunk,
-    CompletionRequest, DeltaToolCall, Function, Tool, VertexRequest, VertexResponse,
+    ChatRequest, Chunk, CompatGenerateRequest, Completion, CompletionComplete, CompletionFinal,
+    CompletionRequest, CompletionType, DeltaToolCall, Function, Prompt, Tool,
 };
-use crate::{FunctionDefinition, ToolCall, ToolType};
+use crate::{FunctionDefinition, HubPreprocessorConfig, ToolCall, ToolChoice, ToolType};
+use crate::{ModelInfo, ModelsInfo};
 use async_stream::__private::AsyncStream;
 use axum::extract::Extension;
-use axum::http::{HeaderMap, Method, StatusCode};
+use axum::http::{HeaderMap, HeaderValue, Method, StatusCode};
 use axum::response::sse::{Event, KeepAlive, Sse};
 use axum::response::{IntoResponse, Response};
 use axum::routing::{get, post};
@@ -30,13 +37,18 @@ use futures::stream::StreamExt;
 use futures::stream::{FuturesOrdered, FuturesUnordered};
 use futures::Stream;
 use futures::TryStreamExt;
+use hf_hub::api::tokio::{Api, ApiBuilder, ApiRepo};
+use hf_hub::{Cache, Repo, RepoType};
+use http::header::AUTHORIZATION;
 use metrics_exporter_prometheus::{Matcher, PrometheusBuilder, PrometheusHandle};
+use pyo3::types::IntoPyDict;
 use serde_json::Value;
 use std::convert::Infallible;
-use std::net::SocketAddr;
-use std::sync::atomic::AtomicBool;
-use std::sync::Arc;
-use text_generation_client::{ShardInfo, ShardedClient};
+use std::fs::File;
+use std::io::BufReader;
+use std::net::{IpAddr, Ipv4Addr, SocketAddr};
+use std::path::{Path, PathBuf};
+use thiserror::Error;
 use tokenizers::Tokenizer;
 use tokio::select;
 use tokio::signal;
@@ -105,6 +117,81 @@ async fn get_model_info(info: Extension<Info>) -> Json<Info> {
     Json(info.0)
 }
 
+#[utoipa::path(
+get,
+tag = "Text Generation Inference",
+path = "/v1/models",
+responses(
+(status = 200, description = "Served model info", body = ModelInfo),
+(status = 404, description = "Model not found", body = ErrorResponse),
+)
+)]
+#[instrument(skip(info))]
+/// Get model info
+async fn openai_get_model_info(info: Extension<Info>) -> Json<ModelsInfo> {
+    Json(ModelsInfo {
+        data: vec![ModelInfo {
+            id: info.0.model_id.clone(),
+            object: "model".to_string(),
+            created: 0, // TODO: determine how to get this
+            owned_by: info.0.model_id.clone(),
+        }],
+        ..Default::default()
+    })
+}
+
+#[utoipa::path(
+    post,
+    tag = "Text Generation Inference",
+    path = "/chat_tokenize",
+    request_body = ChatRequest,
+    responses((status = 200, description = "Templated and tokenized ChatRequest", body = ChatTokenizeResponse))
+)]
+async fn get_chat_tokenize(
+    Extension(infer): Extension<Infer>,
+    Json(chat): Json<ChatRequest>,
+) -> Result<(HeaderMap, Json<ChatTokenizeResponse>), (StatusCode, Json<ErrorResponse>)> {
+    metrics::counter!("tgi_request_count").increment(1);
+
+    let generate_request: GenerateRequest = chat.try_into_generate(&infer)?.0;
+    let input = generate_request.inputs.clone();
+    let encoding = infer.tokenize(generate_request).await?;
+    if let Some(encoding) = encoding {
+        let tokens: Vec<SimpleToken> = encoding
+            .get_ids()
+            .iter()
+            .zip(encoding.get_offsets())
+            .map(|(&id, &(start, stop))| {
+                let text = input
+                    .chars()
+                    .skip(start)
+                    .take(stop - start)
+                    .collect::<String>();
+                SimpleToken {
+                    id,
+                    text,
+                    start,
+                    stop,
+                }
+            })
+            .collect();
+
+        let resp = ChatTokenizeResponse {
+            tokenize_response: TokenizeResponse(tokens),
+            templated_text: input,
+        };
+        Ok((HeaderMap::new(), Json(resp)))
+    } else {
+        Err((
+            StatusCode::NOT_FOUND,
+            Json(ErrorResponse {
+                error: "No fast tokenizer or tokenizer.json for this model".to_string(),
+                error_type: "no fast tokenizer".to_string(),
+            }),
+        ))
+    }
+}
+
 #[utoipa::path(
 get,
 tag = "Text Generation Inference",
@@ -115,10 +202,10 @@ responses(
 example = json ! ({"error": "unhealthy", "error_type": "healthcheck"})),
 )
 )]
-#[instrument(skip(health))]
+#[instrument(skip(infer))]
 /// Health check method
-async fn health(mut health: Extension<Health>) -> Result<(), (StatusCode, Json<ErrorResponse>)> {
-    match health.check().await {
+async fn health(infer: Extension<Infer>) -> Result<(), (StatusCode, Json<ErrorResponse>)> {
+    match infer.health().await {
         true => Ok(()),
         false => Err((
             StatusCode::SERVICE_UNAVAILABLE,
@@ -169,17 +256,20 @@ async fn generate(
     generate_internal(infer, ComputeType(compute_type), Json(req), span).await
 }
 
-async fn generate_internal(
+pub(crate) async fn generate_internal(
     infer: Extension<Infer>,
     ComputeType(compute_type): ComputeType,
     Json(req): Json<GenerateRequest>,
     span: tracing::Span,
 ) -> Result<(HeaderMap, Json<GenerateResponse>), (StatusCode, Json<ErrorResponse>)> {
     let start_time = Instant::now();
-    metrics::increment_counter!("tgi_request_count");
+    metrics::counter!("tgi_request_count").increment(1);
 
     // Do not long ultra long inputs, like image payloads.
-    tracing::debug!("Input: {}", &req.inputs[..1000.min(req.inputs.len())]);
+    tracing::debug!(
+        "Input: {}",
+        &req.inputs.chars().take(1000).collect::<String>()
+    );
 
     let compute_characters = req.inputs.chars().count();
     let mut add_prompt = None;
@@ -215,9 +305,7 @@ async fn generate_internal(
 
                         BestOfSequence {
                             generated_text: output_text,
-                            finish_reason: FinishReason::from(
-                                response.generated_text.finish_reason,
-                            ),
+                            finish_reason: response.generated_text.finish_reason,
                             generated_tokens: response.generated_text.generated_tokens,
                             prefill: response.prefill,
                             tokens: response.tokens,
@@ -229,7 +317,7 @@ async fn generate_internal(
             });
 
             Some(Details {
-                finish_reason: FinishReason::from(response.generated_text.finish_reason),
+                finish_reason: response.generated_text.finish_reason,
                 generated_tokens: response.generated_text.generated_tokens,
                 prefill: response.prefill,
                 tokens: response.tokens,
@@ -294,25 +382,15 @@ async fn generate_internal(
     );
 
     // Metrics
-    metrics::increment_counter!("tgi_request_success");
-    metrics::histogram!("tgi_request_duration", total_time.as_secs_f64());
-    metrics::histogram!(
-        "tgi_request_validation_duration",
-        validation_time.as_secs_f64()
-    );
-    metrics::histogram!("tgi_request_queue_duration", queue_time.as_secs_f64());
-    metrics::histogram!(
-        "tgi_request_inference_duration",
-        inference_time.as_secs_f64()
-    );
-    metrics::histogram!(
-        "tgi_request_mean_time_per_token_duration",
-        time_per_token.as_secs_f64()
-    );
-    metrics::histogram!(
-        "tgi_request_generated_tokens",
-        response.generated_text.generated_tokens as f64
-    );
+    metrics::counter!("tgi_request_success").increment(1);
+    metrics::histogram!("tgi_request_duration").record(total_time.as_secs_f64());
+    metrics::histogram!("tgi_request_validation_duration").record(validation_time.as_secs_f64());
+    metrics::histogram!("tgi_request_queue_duration").record(queue_time.as_secs_f64());
+    metrics::histogram!("tgi_request_inference_duration").record(inference_time.as_secs_f64());
+    metrics::histogram!("tgi_request_mean_time_per_token_duration")
+        .record(time_per_token.as_secs_f64());
+    metrics::histogram!("tgi_request_generated_tokens")
+        .record(response.generated_text.generated_tokens as f64);
 
     // Send response
     let mut output_text = response.generated_text.text;
@@ -392,7 +470,7 @@ async fn generate_stream_internal(
     span: tracing::Span,
 ) -> (HeaderMap, impl Stream<Item = Result<Event, Infallible>>) {
     let start_time = Instant::now();
-    metrics::increment_counter!("tgi_request_count");
+    metrics::counter!("tgi_request_count").increment(1);
 
     tracing::debug!("Input: {}", req.inputs);
 
@@ -420,19 +498,20 @@ async fn generate_stream_internal(
         let best_of = req.parameters.best_of.unwrap_or(1);
         if best_of != 1 {
             let err = InferError::from(ValidationError::BestOfStream);
-            metrics::increment_counter!("tgi_request_failure", "err" => "validation");
+            metrics::counter!("tgi_request_failure", "err" => "validation").increment(1);
             tracing::error!("{err}");
             yield Ok(Event::from(err));
         } else if req.parameters.decoder_input_details {
             let err = InferError::from(ValidationError::PrefillDetailsStream);
-            metrics::increment_counter!("tgi_request_failure", "err" => "validation");
+            metrics::counter!("tgi_request_failure", "err" => "validation").increment(1);
             tracing::error!("{err}");
             yield Ok(Event::from(err));
         } else {
             match infer.generate_stream(req).instrument(info_span!(parent: &span, "async_stream")).await {
                 // Keep permit as long as generate_stream lives
-                Ok((_permit, _input_length, mut response_stream)) => {
+                Ok((_permit, input_length, response_stream)) => {
                     let mut index = 0;
+                    let mut response_stream = Box::pin(response_stream);
                     // Server-Sent Event stream
                     while let Some(response) = response_stream.next().await {
                         index += 1;
@@ -470,9 +549,10 @@ async fn generate_stream_internal(
                                         // Token details
                                         let details = match details {
                                             true => Some(StreamDetails {
-                                                finish_reason: FinishReason::from(generated_text.finish_reason),
+                                                finish_reason: generated_text.finish_reason,
                                                 generated_tokens: generated_text.generated_tokens,
                                                 seed: generated_text.seed,
+                                                input_length,
                                             }),
                                             false => None,
                                         };
@@ -493,13 +573,13 @@ async fn generate_stream_internal(
                                         span.record("seed", format!("{:?}", generated_text.seed));
 
                                         // Metrics
-                                        metrics::increment_counter!("tgi_request_success");
-                                        metrics::histogram!("tgi_request_duration", total_time.as_secs_f64());
-                                        metrics::histogram!("tgi_request_validation_duration", validation_time.as_secs_f64());
-                                        metrics::histogram!("tgi_request_queue_duration", queue_time.as_secs_f64());
-                                        metrics::histogram!("tgi_request_inference_duration", inference_time.as_secs_f64());
-                                        metrics::histogram!("tgi_request_mean_time_per_token_duration", time_per_token.as_secs_f64());
-                                        metrics::histogram!("tgi_request_generated_tokens", generated_text.generated_tokens as f64);
+                                        metrics::counter!("tgi_request_success").increment(1);
+                                        metrics::histogram!("tgi_request_duration").record(total_time.as_secs_f64());
+                                        metrics::histogram!("tgi_request_validation_duration").record(validation_time.as_secs_f64());
+                                        metrics::histogram!("tgi_request_queue_duration").record(queue_time.as_secs_f64());
+                                        metrics::histogram!("tgi_request_inference_duration").record(inference_time.as_secs_f64());
+                                        metrics::histogram!("tgi_request_mean_time_per_token_duration").record(time_per_token.as_secs_f64());
+                                        metrics::histogram!("tgi_request_generated_tokens").record(generated_text.generated_tokens as f64);
 
                                         // StreamResponse
                                         end_reached = true;
@@ -545,8 +625,8 @@ async fn generate_stream_internal(
             // Check if generation reached the end
             // Skip if we already sent an error
             if !end_reached && !error {
-                let err = InferError::IncompleteGeneration;
-                metrics::increment_counter!("tgi_request_failure", "err" => "incomplete");
+                let err = InferError::IncompleteGenerationStream;
+                metrics::counter!("tgi_request_failure", "err" => "incomplete").increment(1);
                 tracing::error!("{err}");
                 yield Ok(Event::from(err));
             }
@@ -558,38 +638,38 @@ async fn generate_stream_internal(
 
 /// Generate tokens
 #[utoipa::path(
-    post,
-    tag = "Text Generation Inference",
-    path = "/v1/completions",
-    request_body = CompletionRequest,
-    responses(
-    (status = 200, description = "Generated Chat Completion",
-    content(
-    ("application/json" = Completion),
-    ("text/event-stream" = CompletionCompleteChunk),
-    )),
-    (status = 424, description = "Generation Error", body = ErrorResponse,
-    example = json ! ({"error": "Request failed during generation"})),
-    (status = 429, description = "Model is overloaded", body = ErrorResponse,
-    example = json ! ({"error": "Model is overloaded"})),
-    (status = 422, description = "Input validation error", body = ErrorResponse,
-    example = json ! ({"error": "Input validation error"})),
-    (status = 500, description = "Incomplete generation", body = ErrorResponse,
-    example = json ! ({"error": "Incomplete generation"})),
-    )
-    )]
+post,
+tag = "Text Generation Inference",
+path = "/v1/completions",
+request_body = CompletionRequest,
+responses(
+(status = 200, description = "Generated Chat Completion",
+content(
+("application/json" = CompletionFinal),
+("text/event-stream" = Chunk),
+)),
+(status = 424, description = "Generation Error", body = ErrorResponse,
+example = json ! ({"error": "Request failed during generation"})),
+(status = 429, description = "Model is overloaded", body = ErrorResponse,
+example = json ! ({"error": "Model is overloaded"})),
+(status = 422, description = "Input validation error", body = ErrorResponse,
+example = json ! ({"error": "Input validation error"})),
+(status = 500, description = "Incomplete generation", body = ErrorResponse,
+example = json ! ({"error": "Incomplete generation"})),
+)
+)]
 #[instrument(
-    skip_all,
-    fields(
-    // parameters = ? req.parameters,
-    total_time,
-    validation_time,
-    queue_time,
-    inference_time,
-    time_per_token,
-    seed,
-    )
-    )]
+skip_all,
+fields(
+// parameters = ? req.parameters,
+total_time,
+validation_time,
+queue_time,
+inference_time,
+time_per_token,
+seed,
+)
+)]
 async fn completions(
     Extension(infer): Extension<Infer>,
     Extension(compute_type): Extension<ComputeType>,
@@ -597,9 +677,10 @@ async fn completions(
     Json(req): Json<CompletionRequest>,
 ) -> Result<Response, (StatusCode, Json<ErrorResponse>)> {
     let span = tracing::Span::current();
-    metrics::increment_counter!("tgi_request_count");
+    metrics::counter!("tgi_request_count").increment(1);
 
     let CompletionRequest {
+        model,
         max_tokens,
         seed,
         stop,
@@ -617,7 +698,7 @@ async fn completions(
 
     // if suffix is present throw an error
     if req.suffix.is_some() {
-        metrics::increment_counter!("tgi_request_failure", "err" => "validation");
+        metrics::counter!("tgi_request_failure", "err" => "validation").increment(1);
         return Err((
             StatusCode::UNPROCESSABLE_ENTITY,
             Json(ErrorResponse {
@@ -628,8 +709,8 @@ async fn completions(
         ));
     }
 
-    if req.prompt.len() > info.max_client_batch_size {
-        metrics::increment_counter!("tgi_request_failure", "err" => "validation");
+    if req.prompt.0.len() > info.max_client_batch_size {
+        metrics::counter!("tgi_request_failure", "err" => "validation").increment(1);
         return Err((
             StatusCode::UNPROCESSABLE_ENTITY,
             Json(ErrorResponse {
@@ -644,9 +725,11 @@ async fn completions(
 
     let generate_requests: Vec<GenerateRequest> = req
         .prompt
+        .0
         .iter()
         .map(|prompt| GenerateRequest {
             inputs: prompt.to_string(),
+            add_special_tokens: true,
             parameters: GenerateParameters {
                 best_of: None,
                 temperature,
@@ -666,6 +749,7 @@ async fn completions(
                 seed,
                 top_n_tokens: None,
                 grammar: None,
+                adapter_id: model.as_ref().filter(|m| *m != "tgi").map(String::from),
             },
         })
         .collect();
@@ -694,22 +778,46 @@ async fn completions(
                         .unwrap_or_else(|_| std::time::Duration::from_secs(0))
                         .as_secs();
 
-                    event
-                        .json_data(CompletionCompleteChunk {
-                            id: "".to_string(),
-                            object: "text_completion".to_string(),
-                            created: current_time,
+                    let message = match stream_token.details {
+                        Some(details) => {
+                            let completion_tokens = details.generated_tokens;
+                            let prompt_tokens = details.input_length;
+                            let total_tokens = prompt_tokens + completion_tokens;
 
+                            Completion::Final(CompletionFinal {
+                                id: String::new(),
+                                created: current_time,
+                                model: model_id.clone(),
+                                system_fingerprint: system_fingerprint.clone(),
+                                choices: vec![CompletionComplete {
+                                    finish_reason: details.finish_reason.to_string(),
+                                    index: index as u32,
+                                    logprobs: None,
+                                    text: stream_token.token.text,
+                                }],
+                                usage: Usage {
+                                    prompt_tokens,
+                                    completion_tokens,
+                                    total_tokens,
+                                },
+                            })
+                        }
+                        None => Completion::Chunk(Chunk {
+                            id: String::new(),
+                            created: current_time,
                             choices: vec![CompletionComplete {
-                                finish_reason: "".to_string(),
+                                finish_reason: String::new(),
                                 index: index as u32,
                                 logprobs: None,
                                 text: stream_token.token.text,
                             }],
-
                             model: model_id.clone(),
                             system_fingerprint: system_fingerprint.clone(),
-                        })
+                        }),
+                    };
+
+                    event
+                        .json_data(message)
                         .unwrap_or_else(|_e| Event::default())
                 };
 
@@ -811,6 +919,10 @@ async fn completions(
             }
         };
 
+        let stream = stream.chain(futures::stream::once(async {
+            Ok(Event::default().data("[DONE]"))
+        }));
+
         let sse = Sse::new(stream).keep_alive(KeepAlive::default());
         Ok((headers, sse).into_response())
     } else {
@@ -913,7 +1025,7 @@ async fn completions(
                 total_tokens += details.prefill.len() as u32 + details.generated_tokens;
 
                 Ok(CompletionComplete {
-                    finish_reason: details.finish_reason.to_string(),
+                    finish_reason: details.finish_reason.format(true),
                     index: index as u32,
                     logprobs: None,
                     text: generation.generated_text,
@@ -922,9 +1034,8 @@ async fn completions(
             .collect::<Result<Vec<_>, _>>()
             .map_err(|(status, Json(err))| (status, Json(err)))?;
 
-        let response = Completion {
+        let response = Completion::Final(CompletionFinal {
             id: "".to_string(),
-            object: "text_completion".to_string(),
             created: current_time,
             model: info.model_id.clone(),
             system_fingerprint: format!(
@@ -938,7 +1049,7 @@ async fn completions(
                 completion_tokens,
                 total_tokens,
             },
-        };
+        });
 
         // headers similar to `generate` but aggregated
         let mut headers = HeaderMap::new();
@@ -962,135 +1073,56 @@ async fn completions(
 
 /// Generate tokens
 #[utoipa::path(
-    post,
-    tag = "Text Generation Inference",
-    path = "/v1/chat/completions",
-    request_body = ChatRequest,
-    responses(
-    (status = 200, description = "Generated Chat Completion",
-    content(
-    ("application/json" = ChatCompletion),
-    ("text/event-stream" = ChatCompletionChunk),
-    )),
-    (status = 424, description = "Generation Error", body = ErrorResponse,
-    example = json ! ({"error": "Request failed during generation"})),
-    (status = 429, description = "Model is overloaded", body = ErrorResponse,
-    example = json ! ({"error": "Model is overloaded"})),
-    (status = 422, description = "Input validation error", body = ErrorResponse,
-    example = json ! ({"error": "Input validation error"})),
-    (status = 500, description = "Incomplete generation", body = ErrorResponse,
-    example = json ! ({"error": "Incomplete generation"})),
-    )
-    )]
+post,
+tag = "Text Generation Inference",
+path = "/v1/chat/completions",
+request_body = ChatRequest,
+responses(
+(status = 200, description = "Generated Chat Completion",
+content(
+("application/json" = ChatCompletion),
+("text/event-stream" = ChatCompletionChunk),
+)),
+(status = 424, description = "Generation Error", body = ErrorResponse,
+example = json ! ({"error": "Request failed during generation"})),
+(status = 429, description = "Model is overloaded", body = ErrorResponse,
+example = json ! ({"error": "Model is overloaded"})),
+(status = 422, description = "Input validation error", body = ErrorResponse,
+example = json ! ({"error": "Input validation error"})),
+(status = 500, description = "Incomplete generation", body = ErrorResponse,
+example = json ! ({"error": "Incomplete generation"})),
+)
+)]
 #[instrument(
-    skip_all,
-    fields(
-    // parameters = ? req.parameters,
-    total_time,
-    validation_time,
-    queue_time,
-    inference_time,
-    time_per_token,
-    seed,
-    )
-    )]
+skip_all,
+fields(
+// parameters = ? req.parameters,
+total_time,
+validation_time,
+queue_time,
+inference_time,
+time_per_token,
+seed,
+)
+)]
 async fn chat_completions(
     Extension(infer): Extension<Infer>,
     Extension(compute_type): Extension<ComputeType>,
     Extension(info): Extension<Info>,
-    Json(req): Json<ChatRequest>,
+    Json(chat): Json<ChatRequest>,
 ) -> Result<Response, (StatusCode, Json<ErrorResponse>)> {
     let span = tracing::Span::current();
-    metrics::increment_counter!("tgi_request_count");
+    metrics::counter!("tgi_request_count").increment(1);
     let ChatRequest {
-        logprobs,
-        max_tokens,
-        messages,
-        presence_penalty,
-        seed,
-        stop,
         stream,
-        tools,
-        tool_choice,
-        tool_prompt,
-        temperature,
+        stream_options,
+        logprobs,
         ..
-    } = req;
+    } = chat.clone();
+    let (generate_request, using_tools): (GenerateRequest, bool) =
+        chat.try_into_generate(&infer)?;
 
-    let repetition_penalty = presence_penalty.map(|x| x + 2.0);
-    let logprobs = logprobs.unwrap_or(false);
-    let tool_prompt = tool_prompt.unwrap_or_default();
-    let stop = stop.unwrap_or_default();
-    // enable greedy only when temperature is 0
-    let (do_sample, temperature) = match temperature {
-        Some(temperature) if temperature == 0.0 => (false, None),
-        other => (true, other),
-    };
-
-    // extract tool grammar if present
-    let tool_grammar = match ToolGrammar::apply(tools, tool_choice) {
-        Ok(grammar) => grammar,
-        Err(err) => {
-            metrics::increment_counter!("tgi_request_failure", "err" => "validation");
-            tracing::error!("{err}");
-            return Err((
-                StatusCode::UNPROCESSABLE_ENTITY,
-                Json(ErrorResponse {
-                    error: err.to_string(),
-                    error_type: err.error_type().to_string(),
-                }),
-            ));
-        }
-    };
-
-    let grammar_with_prompt = tool_grammar
-        .as_ref()
-        .map(|t| (GrammarType::Json(serde_json::json!(t)), tool_prompt));
-
-    let typed_grammar = grammar_with_prompt
-        .as_ref()
-        .map(|(grammar, _)| grammar.clone());
-
-    // apply chat template to flatten the request into a single input
-    let inputs = match infer.apply_chat_template(messages, grammar_with_prompt) {
-        Ok(inputs) => inputs,
-        Err(err) => {
-            metrics::increment_counter!("tgi_request_failure", "err" => "validation");
-            tracing::error!("{err}");
-            return Err((
-                StatusCode::UNPROCESSABLE_ENTITY,
-                Json(ErrorResponse {
-                    error: err.to_string(),
-                    error_type: err.error_type().to_string(),
-                }),
-            ));
-        }
-    };
-
-    // build the request passing some parameters
-    let generate_request = GenerateRequest {
-        inputs: inputs.to_string(),
-        parameters: GenerateParameters {
-            best_of: None,
-            temperature,
-            repetition_penalty,
-            frequency_penalty: req.frequency_penalty,
-            top_k: None,
-            top_p: req.top_p,
-            typical_p: None,
-            do_sample,
-            max_new_tokens: max_tokens,
-            return_full_text: None,
-            stop,
-            truncate: None,
-            watermark: false,
-            details: true,
-            decoder_input_details: !stream,
-            seed,
-            top_n_tokens: req.top_logprobs,
-            grammar: typed_grammar,
-        },
-    };
+    let logprobs = logprobs.unwrap_or_default();
 
     // static values that will be returned in all cases
     let model_id = info.model_id.clone();
@@ -1112,7 +1144,7 @@ async fn chat_completions(
             });
 
             // replace the content with the tool calls if grammar is present
-            let (content, tool_calls) = if tool_grammar.is_some() {
+            let (content, tool_calls) = if using_tools {
                 (None, Some(vec![stream_token.token.text]))
             } else {
                 let content = if !stream_token.token.special {
@@ -1124,15 +1156,40 @@ async fn chat_completions(
                 (content, None)
             };
 
+            let (usage, finish_reason) = match stream_token.details {
+                Some(details) => {
+                    let usage = if stream_options
+                        .as_ref()
+                        .map(|s| s.include_usage)
+                        .unwrap_or(false)
+                    {
+                        let completion_tokens = details.generated_tokens;
+                        let prompt_tokens = details.input_length;
+                        let total_tokens = prompt_tokens + completion_tokens;
+                        Some(Usage {
+                            completion_tokens,
+                            prompt_tokens,
+                            total_tokens,
+                        })
+                    } else {
+                        None
+                    };
+                    (usage, Some(details.finish_reason.format(true)))
+                }
+                None => (None, None),
+            };
             event
-                .json_data(ChatCompletionChunk::new(
-                    model_id.clone(),
-                    system_fingerprint.clone(),
-                    content,
-                    tool_calls,
-                    current_time,
-                    logprobs,
-                    stream_token.details.map(|d| d.finish_reason.to_string()),
+                .json_data(CompletionType::ChatCompletionChunk(
+                    ChatCompletionChunk::new(
+                        model_id.clone(),
+                        system_fingerprint.clone(),
+                        content,
+                        tool_calls,
+                        current_time,
+                        logprobs,
+                        finish_reason,
+                        usage,
+                    ),
                 ))
                 .unwrap_or_else(|e| {
                     println!("Failed to serialize ChatCompletionChunk: {:?}", e);
@@ -1148,6 +1205,11 @@ async fn chat_completions(
             span,
         )
         .await;
+
+        let response_stream = response_stream.chain(futures::stream::once(async {
+            Ok(Event::default().data("[DONE]"))
+        }));
+
         let sse = Sse::new(response_stream).keep_alive(KeepAlive::default());
         Ok((headers, sse).into_response())
     } else {
@@ -1159,48 +1221,62 @@ async fn chat_completions(
             .unwrap_or_else(|_| std::time::Duration::from_secs(0))
             .as_secs();
 
-        let (tool_calls, output) = if tool_grammar.is_some() {
-            // gen_text should be valid json
+        let (tool_calls, output) = if using_tools {
             let gen_text_value: Value =
                 serde_json::from_str(&generation.generated_text).map_err(|e| {
-                    (
-                        StatusCode::UNPROCESSABLE_ENTITY,
-                        Json(ErrorResponse {
-                            error: e.to_string(),
-                            error_type: "Input validation error".to_string(),
-                        }),
-                    )
+                    InferError::ToolError(format!(
+                        "Failed to parse generated text: {} {:?}",
+                        e, generation.generated_text
+                    ))
                 })?;
-            let tool_calls = vec![ToolCall {
-                id: "0".to_string(),
-                r#type: "function".to_string(),
-                function: FunctionDefinition {
-                    description: None,
-                    name: gen_text_value
-                        .get("function")
-                        .and_then(|f| f.get("_name"))
-                        .and_then(|name| name.as_str())
-                        .unwrap_or("default_function_name")
-                        .to_string(),
-                    // Serialize the JSON object obtained from "function" to an escaped JSON string
-                    arguments: gen_text_value
-                        .get("function")
-                        .map(|f| {
-                            let mut f_cloned = f.clone();
-                            if let Value::Object(ref mut props) = f_cloned {
-                                props.remove("_name");
-                            }
-                            f_cloned
-                        })
-                        .unwrap_or_default(),
-                },
-            }];
-            (Some(tool_calls), None)
+            let function = gen_text_value.get("function").ok_or(InferError::ToolError(
+                "No function found in generated text".to_string(),
+            ))?;
+
+            let name = function
+                .get("_name")
+                .and_then(Value::as_str)
+                .ok_or(InferError::ToolError(
+                    "No _name found in generated text".to_string(),
+                ))?
+                .to_string();
+
+            let mut arguments = function.clone();
+            if let Value::Object(ref mut props) = arguments {
+                props.remove("_name");
+            }
+            match name.as_str() {
+                "notify_error" => {
+                    // parse the error message
+                    let error_message = arguments
+                        .get("error")
+                        .and_then(Value::as_str)
+                        .ok_or_else(|| {
+                            InferError::ToolError(
+                                "No error message found in generated text".to_string(),
+                            )
+                        })?
+                        .to_string();
+                    (None, Some(error_message))
+                }
+                _ => {
+                    let tool_calls = vec![ToolCall {
+                        id: "0".to_string(),
+                        r#type: "function".to_string(),
+                        function: FunctionDefinition {
+                            description: None,
+                            name,
+                            arguments,
+                        },
+                    }];
+                    (Some(tool_calls), None)
+                }
+            }
         } else {
             (None, Some(generation.generated_text))
         };
         // build the complete response object with the full text
-        let response = ChatCompletion::new(
+        let response = CompletionType::ChatCompletion(ChatCompletion::new(
             model_id,
             system_fingerprint,
             output,
@@ -1208,118 +1284,25 @@ async fn chat_completions(
             generation.details.unwrap(),
             logprobs,
             tool_calls,
-        );
+        ));
 
         // wrap generation inside a Vec to match api-inference
         Ok((headers, Json(response)).into_response())
     }
 }
 
-/// Generate tokens from Vertex request
-#[utoipa::path(
-    post,
-    tag = "Text Generation Inference",
-    path = "/vertex",
-    request_body = VertexRequest,
-    responses(
-    (status = 200, description = "Generated Text", body = VertexResponse),
-    (status = 424, description = "Generation Error", body = ErrorResponse,
-    example = json ! ({"error": "Request failed during generation"})),
-    (status = 429, description = "Model is overloaded", body = ErrorResponse,
-    example = json ! ({"error": "Model is overloaded"})),
-    (status = 422, description = "Input validation error", body = ErrorResponse,
-    example = json ! ({"error": "Input validation error"})),
-    (status = 500, description = "Incomplete generation", body = ErrorResponse,
-    example = json ! ({"error": "Incomplete generation"})),
-    )
-    )]
-#[instrument(
-    skip_all,
-    fields(
-        total_time,
-        validation_time,
-        queue_time,
-        inference_time,
-        time_per_token,
-        seed,
-    )
-)]
-async fn vertex_compatibility(
-    Extension(infer): Extension<Infer>,
-    Extension(compute_type): Extension<ComputeType>,
-    Json(req): Json<VertexRequest>,
-) -> Result<Response, (StatusCode, Json<ErrorResponse>)> {
-    let span = tracing::Span::current();
-    metrics::increment_counter!("tgi_request_count");
-
-    // check that theres at least one instance
-    if req.instances.is_empty() {
-        return Err((
-            StatusCode::UNPROCESSABLE_ENTITY,
-            Json(ErrorResponse {
-                error: "Input validation error".to_string(),
-                error_type: "Input validation error".to_string(),
-            }),
-        ));
-    }
-
-    // Process all instances
-    let predictions = req
-        .instances
-        .iter()
-        .map(|instance| {
-            let generate_request = GenerateRequest {
-                inputs: instance.inputs.clone(),
-                parameters: GenerateParameters {
-                    do_sample: true,
-                    max_new_tokens: instance.parameters.as_ref().and_then(|p| p.max_new_tokens),
-                    seed: instance.parameters.as_ref().and_then(|p| p.seed),
-                    details: true,
-                    decoder_input_details: true,
-                    ..Default::default()
-                },
-            };
-
-            async {
-                generate_internal(
-                    Extension(infer.clone()),
-                    compute_type.clone(),
-                    Json(generate_request),
-                    span.clone(),
-                )
-                .await
-                .map(|(_, Json(generation))| generation.generated_text)
-                .map_err(|_| {
-                    (
-                        StatusCode::INTERNAL_SERVER_ERROR,
-                        Json(ErrorResponse {
-                            error: "Incomplete generation".into(),
-                            error_type: "Incomplete generation".into(),
-                        }),
-                    )
-                })
-            }
-        })
-        .collect::<FuturesUnordered<_>>()
-        .try_collect::<Vec<_>>()
-        .await?;
-
-    let response = VertexResponse { predictions };
-    Ok((HeaderMap::new(), Json(response)).into_response())
-}
-
 /// Tokenize inputs
 #[utoipa::path(
-    post,
-    tag = "Text Generation Inference",
-    path = "/tokenize",
-    request_body = GenerateRequest,
-    responses(
-    (status = 200, description = "Tokenized ids", body = TokenizeResponse),
-    (status = 404, description = "No tokenizer found", body = ErrorResponse,
-    example = json ! ({"error": "No fast tokenizer available"})),
-    )
-    )]
+post,
+tag = "Text Generation Inference",
+path = "/tokenize",
+request_body = GenerateRequest,
+responses(
+(status = 200, description = "Tokenized ids", body = TokenizeResponse),
+(status = 404, description = "No tokenizer found", body = ErrorResponse,
+example = json ! ({"error": "No fast tokenizer available"})),
+)
+)]
 #[instrument(skip_all)]
 async fn tokenize(
     Extension(infer): Extension<Infer>,
@@ -1333,7 +1316,11 @@ async fn tokenize(
             .iter()
             .zip(encoding.get_offsets())
             .map(|(&id, &(start, stop))| {
-                let text: String = input.chars().skip(start).take(stop - start).collect();
+                let text = input
+                    .chars()
+                    .skip(start)
+                    .take(stop - start)
+                    .collect::<String>();
                 SimpleToken {
                     id,
                     text,
@@ -1356,10 +1343,10 @@ async fn tokenize(
 
 /// Prometheus metrics scrape endpoint
 #[utoipa::path(
-get,
-tag = "Text Generation Inference",
-path = "/metrics",
-responses((status = 200, description = "Prometheus Metrics", body = String))
+    get,
+    tag = "Text Generation Inference",
+    path = "/metrics",
+    responses((status = 200, description = "Prometheus Metrics", body = String))
 )]
 async fn metrics(prom_handle: Extension<PrometheusHandle>) -> String {
     prom_handle.render()
@@ -1368,135 +1355,489 @@ async fn metrics(prom_handle: Extension<PrometheusHandle>) -> String {
 #[derive(Clone, Debug)]
 pub(crate) struct ComputeType(String);
 
+// OpenAPI documentation
+#[derive(OpenApi)]
+#[openapi(
+paths(
+health,
+get_model_info,
+compat_generate,
+generate,
+generate_stream,
+chat_completions,
+completions,
+tokenize,
+metrics,
+openai_get_model_info,
+),
+components(
+schemas(
+Info,
+CompatGenerateRequest,
+GenerateRequest,
+GrammarType,
+ChatRequest,
+Message,
+MessageContent,
+MessageChunk,
+Url,
+FunctionName,
+OutputMessage,
+TextMessage,
+ToolCallMessage,
+ToolCallDelta,
+ChatCompletionComplete,
+ChatCompletionChoice,
+ChatCompletionDelta,
+ChatCompletionChunk,
+ChatCompletionLogprob,
+ChatCompletionLogprobs,
+ChatCompletionTopLogprob,
+ChatCompletion,
+CompletionRequest,
+CompletionComplete,
+Chunk,
+Completion,
+CompletionFinal,
+Prompt,
+GenerateParameters,
+PrefillToken,
+Token,
+GenerateResponse,
+TokenizeResponse,
+SimpleToken,
+BestOfSequence,
+Details,
+FinishReason,
+StreamResponse,
+StreamDetails,
+ErrorResponse,
+GrammarType,
+Usage,
+StreamOptions,
+DeltaToolCall,
+ToolType,
+Tool,
+ToolCall,
+Function,
+FunctionDefinition,
+ToolChoice,
+ModelInfo,
+)
+),
+tags(
+(name = "Text Generation Inference", description = "Hugging Face Text Generation Inference API")
+),
+info(
+title = "Text Generation Inference",
+license(
+name = "Apache 2.0",
+url = "https://www.apache.org/licenses/LICENSE-2.0"
+)
+)
+)]
+pub struct ApiDoc;
+
+pub fn schema() -> ApiDoc {
+    ApiDoc
+}
+
 /// Serving method
 #[allow(clippy::too_many_arguments)]
 pub async fn run(
-    model_info: HubModelInfo,
-    shard_info: ShardInfo,
-    compat_return_full_text: bool,
+    backend: impl Backend + Send + Sync + 'static,
     max_concurrent_requests: usize,
     max_best_of: usize,
     max_stop_sequences: usize,
     max_top_n_tokens: u32,
-    max_input_length: usize,
+    max_input_tokens: usize,
     max_total_tokens: usize,
-    waiting_served_ratio: f32,
-    max_batch_prefill_tokens: u32,
-    max_batch_total_tokens: u32,
-    max_waiting_tokens: usize,
-    max_batch_size: Option<usize>,
-    client: ShardedClient,
-    tokenizer: Option<Tokenizer>,
-    config: Option<Config>,
     validation_workers: usize,
-    addr: SocketAddr,
-    allow_origin: Option<AllowOrigin>,
+    api_key: Option<String>,
+    tokenizer_name: String,
+    tokenizer_config_path: Option<String>,
+    revision: Option<String>,
+    hostname: String,
+    port: u16,
+    cors_allow_origin: Option<Vec<String>>,
     ngrok: bool,
-    ngrok_authtoken: Option<String>,
-    ngrok_edge: Option<String>,
-    tokenizer_config: HubTokenizerConfig,
+    _ngrok_authtoken: Option<String>,
+    _ngrok_edge: Option<String>,
     messages_api_enabled: bool,
-    grammar_support: bool,
+    disable_grammar_support: bool,
     max_client_batch_size: usize,
-) -> Result<(), axum::BoxError> {
-    // OpenAPI documentation
-    #[derive(OpenApi)]
-    #[openapi(
-    paths(
-    health,
-    get_model_info,
-    compat_generate,
-    generate,
-    generate_stream,
-    chat_completions,
-    completions,
-    tokenize,
-    metrics,
-    ),
-    components(
-    schemas(
-    Info,
-    CompatGenerateRequest,
-    GenerateRequest,
-    GrammarType,
-    ChatRequest,
-    Message,
-    ChatCompletionComplete,
-    ChatCompletionChoice,
-    ChatCompletionDelta,
-    ChatCompletionChunk,
-    ChatCompletionLogprob,
-    ChatCompletionLogprobs,
-    ChatCompletionTopLogprob,
-    ChatCompletion,
-    CompletionRequest,
-    CompletionComplete,
-    CompletionCompleteChunk,
-    GenerateParameters,
-    PrefillToken,
-    Token,
-    GenerateResponse,
-    TokenizeResponse,
-    SimpleToken,
-    BestOfSequence,
-    Details,
-    FinishReason,
-    StreamResponse,
-    StreamDetails,
-    ErrorResponse,
-    GrammarType,
-    Usage,
-    DeltaToolCall,
-    ToolType,
-    Tool,
-    ToolCall,
-    Function,
-    FunctionDefinition,
+    usage_stats_level: usage_stats::UsageStatsLevel,
+) -> Result<(), WebServerError> {
+    // CORS allowed origins
+    // map to go inside the option and then map to parse from String to HeaderValue
+    // Finally, convert to AllowOrigin
+    let allow_origin: Option<AllowOrigin> = cors_allow_origin.map(|cors_allow_origin| {
+        AllowOrigin::list(
+            cors_allow_origin
+                .iter()
+                .map(|origin| origin.parse::<HeaderValue>().unwrap()),
+        )
+    });
+
+    // Parse Huggingface hub token
+    let authorization_token = std::env::var("HF_TOKEN")
+        .or_else(|_| std::env::var("HUGGING_FACE_HUB_TOKEN"))
+        .ok();
+
+    // Tokenizer instance
+    // This will only be used to validate payloads
+    let local_path = Path::new(&tokenizer_name);
+
+    // Shared API builder initialization
+    let api_builder = || {
+        let mut builder = ApiBuilder::new()
+            .with_progress(false)
+            .with_token(authorization_token);
+
+        if let Ok(cache_dir) = std::env::var("HUGGINGFACE_HUB_CACHE") {
+            builder = builder.with_cache_dir(cache_dir.into());
+        }
+
+        builder
+    };
+
+    // Decide if we need to use the API based on the revision and local path
+    let use_api = revision.is_some() || !local_path.exists() || !local_path.is_dir();
+
+    // Initialize API if needed
+    #[derive(Clone)]
+    enum Type {
+        Api(Api),
+        Cache(Cache),
+        None,
+    }
+    let api = if use_api {
+        if std::env::var("HF_HUB_OFFLINE") == Ok("1".to_string()) {
+            let cache = std::env::var("HUGGINGFACE_HUB_CACHE")
+                .map_err(|_| ())
+                .map(|cache_dir| Cache::new(cache_dir.into()))
+                .unwrap_or_else(|_| Cache::default());
+            tracing::warn!("Offline mode active using cache defaults");
+            Type::Cache(cache)
+        } else {
+            tracing::info!("Using the Hugging Face API");
+            match api_builder().build() {
+                Ok(api) => Type::Api(api),
+                Err(_) => {
+                    tracing::warn!("Unable to build the Hugging Face API");
+                    Type::None
+                }
+            }
+        }
+    } else {
+        Type::None
+    };
+
+    // Load tokenizer and model info
+    let (
+        tokenizer_filename,
+        config_filename,
+        tokenizer_config_filename,
+        preprocessor_config_filename,
+        processor_config_filename,
+        model_info,
+    ) = match api {
+        Type::None => (
+            Some(local_path.join("tokenizer.json")),
+            Some(local_path.join("config.json")),
+            Some(local_path.join("tokenizer_config.json")),
+            Some(local_path.join("preprocessor_config.json")),
+            Some(local_path.join("processor_config.json")),
+            None,
+        ),
+        Type::Api(api) => {
+            let api_repo = api.repo(Repo::with_revision(
+                tokenizer_name.to_string(),
+                RepoType::Model,
+                revision.clone().unwrap_or_else(|| "main".to_string()),
+            ));
+
+            let tokenizer_filename = match api_repo.get("tokenizer.json").await {
+                Ok(tokenizer_filename) => Some(tokenizer_filename),
+                Err(_) => get_base_tokenizer(&api, &api_repo).await,
+            };
+            let config_filename = api_repo.get("config.json").await.ok();
+            let tokenizer_config_filename = api_repo.get("tokenizer_config.json").await.ok();
+            let preprocessor_config_filename = api_repo.get("preprocessor_config.json").await.ok();
+            let processor_config_filename = api_repo.get("processor_config.json").await.ok();
+
+            let model_info = if let Some(model_info) = get_hub_model_info(&api_repo).await {
+                Some(model_info)
+            } else {
+                tracing::warn!("Could not retrieve model info from the Hugging Face hub.");
+                None
+            };
+            (
+                tokenizer_filename,
+                config_filename,
+                tokenizer_config_filename,
+                preprocessor_config_filename,
+                processor_config_filename,
+                model_info,
+            )
+        }
+        Type::Cache(cache) => {
+            let repo = cache.repo(Repo::with_revision(
+                tokenizer_name.to_string(),
+                RepoType::Model,
+                revision.clone().unwrap_or_else(|| "main".to_string()),
+            ));
+            (
+                repo.get("tokenizer.json"),
+                repo.get("config.json"),
+                repo.get("tokenizer_config.json"),
+                repo.get("preprocessor_config.json"),
+                repo.get("processor_config.json"),
+                None,
+            )
+        }
+    };
+
+    // Read the JSON contents of the file as an instance of 'HubTokenizerConfig'.
+    let tokenizer_config: Option<HubTokenizerConfig> = if let Some(filename) = tokenizer_config_path
+    {
+        HubTokenizerConfig::from_file(filename)
+    } else {
+        tokenizer_config_filename.and_then(HubTokenizerConfig::from_file)
+    };
+    let tokenizer_config = tokenizer_config.unwrap_or_else(|| {
+        tracing::warn!("Could not find tokenizer config locally and no API specified");
+        HubTokenizerConfig::default()
+    });
+
+    let tokenizer: Option<Tokenizer> = tokenizer_filename.and_then(|filename| {
+        use pyo3::prelude::*;
+        let convert = pyo3::Python::with_gil(|py| -> PyResult<()> {
+            let transformers = py.import_bound("transformers")?;
+            let auto = transformers.getattr("AutoTokenizer")?;
+            let from_pretrained = auto.getattr("from_pretrained")?;
+            let args = (tokenizer_name.to_string(),);
+            let kwargs = [(
+                "revision",
+                revision.clone().unwrap_or_else(|| "main".to_string()),
+            )]
+            .into_py_dict_bound(py);
+            let tokenizer = from_pretrained.call(args, Some(&kwargs))?;
+            let save = tokenizer.getattr("save_pretrained")?;
+            let args = ("out".to_string(),);
+            save.call1(args)?;
+            Ok(())
+        });
+        //.inspect_err(|err| {
+        //    tracing::error!("Failed to import python tokenizer {err}");
+        //});
+        let filename = if convert.is_ok() {
+            // If we have correctly loaded and resaved with transformers
+            // We might have modified the tokenizer.json according to transformers
+            "out/tokenizer.json".into()
+        } else {
+            filename
+        };
+        Tokenizer::from_file(filename).ok()
+    });
+
+    let config: Option<Config> = config_filename.and_then(|filename| {
+        std::fs::read_to_string(filename)
+            .ok()
+            .as_ref()
+            .and_then(|c| {
+                let config: Result<Config, _> = serde_json::from_str(c);
+                if let Err(err) = &config {
+                    tracing::warn!("Could not parse config {err:?}");
+                }
+                config.ok()
+            })
+    });
+    let model_info = model_info.unwrap_or_else(|| HubModelInfo {
+        model_id: tokenizer_name.to_string(),
+        sha: None,
+        pipeline_tag: None,
+    });
+
+    let processor_config = processor_config_filename
+        .and_then(HubProcessorConfig::from_file)
+        .unwrap_or_default();
+
+    let preprocessor_config: Option<HubPreprocessorConfig> =
+        preprocessor_config_filename.and_then(HubPreprocessorConfig::from_file);
+
+    tracing::info!("Using config {config:?}");
+    if tokenizer.is_none() {
+        tracing::warn!("Could not find a fast tokenizer implementation for {tokenizer_name}");
+        tracing::warn!("Rust input length validation and truncation is disabled");
+    }
+
+    // Only send usage stats when TGI is run in container and the function returns Some
+    let is_container = matches!(usage_stats::is_container(), Ok(true));
+    let user_agent = match (usage_stats_level, is_container) {
+        (usage_stats::UsageStatsLevel::On | usage_stats::UsageStatsLevel::NoStack, true) => {
+            let reduced_args = usage_stats::Args::new(
+                config.clone(),
+                tokenizer_config.tokenizer_class.clone(),
+                max_concurrent_requests,
+                max_best_of,
+                max_stop_sequences,
+                max_top_n_tokens,
+                max_input_tokens,
+                max_total_tokens,
+                // waiting_served_ratio,
+                // max_batch_prefill_tokens,
+                // max_batch_total_tokens,
+                // max_waiting_tokens,
+                // max_batch_size,
+                revision.clone(),
+                validation_workers,
+                messages_api_enabled,
+                disable_grammar_support,
+                max_client_batch_size,
+                usage_stats_level,
+            );
+            Some(usage_stats::UserAgent::new(reduced_args))
+        }
+        _ => None,
+    };
+
+    if let Some(ref ua) = user_agent {
+        let start_event =
+            usage_stats::UsageStatsEvent::new(ua.clone(), usage_stats::EventType::Start, None);
+        tokio::spawn(async move {
+            start_event.send().await;
+        });
+    };
+    let compat_return_full_text = match &model_info.pipeline_tag {
+        None => {
+            tracing::warn!("no pipeline tag found for model {tokenizer_name}");
+            true
+        }
+        Some(pipeline_tag) => pipeline_tag.as_str() == "text-generation",
+    };
+    let result = start(
+        backend,
+        max_concurrent_requests,
+        max_best_of,
+        max_stop_sequences,
+        max_top_n_tokens,
+        max_input_tokens,
+        max_total_tokens,
+        validation_workers,
+        api_key,
+        config,
+        (tokenizer, tokenizer_config),
+        (preprocessor_config, processor_config),
+        hostname,
+        port,
+        ngrok,
+        _ngrok_authtoken,
+        _ngrok_edge,
+        messages_api_enabled,
+        disable_grammar_support,
+        max_client_batch_size,
+        model_info,
+        compat_return_full_text,
+        allow_origin,
     )
-    ),
-    tags(
-    (name = "Text Generation Inference", description = "Hugging Face Text Generation Inference API")
-    ),
-    info(
-    title = "Text Generation Inference",
-    license(
-    name = "Apache 2.0",
-    url = "https://www.apache.org/licenses/LICENSE-2.0"
-    )
-    )
-    )]
-    struct ApiDoc;
+    .await;
+
+    if let Some(ua) = user_agent {
+        match result {
+            Ok(_) => {
+                let stop_event = usage_stats::UsageStatsEvent::new(
+                    ua.clone(),
+                    usage_stats::EventType::Stop,
+                    None,
+                );
+                stop_event.send().await;
+                Ok(())
+            }
+            Err(e) => {
+                let description = match usage_stats_level {
+                    usage_stats::UsageStatsLevel::On => Some(e.to_string()),
+                    usage_stats::UsageStatsLevel::NoStack => Some("unknow_error".to_string()),
+                    _ => None,
+                };
+                let event = usage_stats::UsageStatsEvent::new(
+                    ua.clone(),
+                    usage_stats::EventType::Error,
+                    description,
+                );
+                event.send().await;
+
+                Err(e)
+            }
+        }
+    } else {
+        result
+    }
+}
+
+#[allow(clippy::too_many_arguments)]
+async fn start(
+    backend: impl Backend + Send + Sync + 'static,
+    max_concurrent_requests: usize,
+    max_best_of: usize,
+    max_stop_sequences: usize,
+    max_top_n_tokens: u32,
+    max_input_tokens: usize,
+    max_total_tokens: usize,
+    validation_workers: usize,
+    api_key: Option<String>,
+    config: Option<Config>,
+    (tokenizer, tokenizer_config): (Option<Tokenizer>, HubTokenizerConfig),
+    (preprocessor_config, processor_config): (Option<HubPreprocessorConfig>, HubProcessorConfig),
+    hostname: String,
+    port: u16,
+    ngrok: bool,
+    _ngrok_authtoken: Option<String>,
+    _ngrok_edge: Option<String>,
+    messages_api_enabled: bool,
+    disable_grammar_support: bool,
+    max_client_batch_size: usize,
+    model_info: HubModelInfo,
+    compat_return_full_text: bool,
+    allow_origin: Option<AllowOrigin>,
+) -> Result<(), WebServerError> {
+    // Determine the server port based on the feature and environment variable.
+    let port = if cfg!(feature = "google") {
+        std::env::var("AIP_HTTP_PORT")
+            .map(|aip_http_port| aip_http_port.parse::<u16>().unwrap_or(port))
+            .unwrap_or(port)
+    } else {
+        port
+    };
+
+    let addr = match hostname.parse() {
+        Ok(ip) => SocketAddr::new(ip, port),
+        Err(_) => {
+            tracing::warn!("Invalid hostname, defaulting to 0.0.0.0");
+            SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), port)
+        }
+    };
 
     // Create state
     let validation = Validation::new(
         validation_workers,
         tokenizer,
         config,
+        preprocessor_config,
         max_best_of,
         max_stop_sequences,
         max_top_n_tokens,
-        max_input_length,
+        max_input_tokens,
         max_total_tokens,
-        grammar_support,
+        disable_grammar_support,
     );
-    let generation_health = Arc::new(AtomicBool::new(false));
-    let health_ext = Health::new(client.clone(), generation_health.clone());
+
     let infer = Infer::new(
-        client,
+        backend,
         validation,
-        waiting_served_ratio,
-        max_batch_prefill_tokens,
-        max_batch_total_tokens,
-        max_waiting_tokens,
-        max_batch_size,
         max_concurrent_requests,
-        shard_info.requires_padding,
-        max_input_length as u32,
-        max_total_tokens as u32,
-        shard_info.window_size,
-        shard_info.speculate,
-        generation_health,
         tokenizer_config,
+        processor_config,
     );
 
     // Duration buckets
@@ -1513,7 +1854,7 @@ pub async fn run(
     // Input Length buckets
     let input_length_matcher = Matcher::Full(String::from("tgi_request_input_length"));
     let input_length_buckets: Vec<f64> = (0..100)
-        .map(|x| (max_input_length as f64 / 100.0) * (x + 1) as f64)
+        .map(|x| (max_input_tokens as f64 / 100.0) * (x + 1) as f64)
         .collect();
     // Generated tokens buckets
     let generated_tokens_matcher = Matcher::Full(String::from("tgi_request_generated_tokens"));
@@ -1529,8 +1870,8 @@ pub async fn run(
     let batch_size_matcher = Matcher::Full(String::from("tgi_batch_next_size"));
     let batch_size_buckets: Vec<f64> = (0..1024).map(|x| (x + 1) as f64).collect();
     // Speculated tokens buckets
-    let skipped_matcher = Matcher::Full(String::from("tgi_request_skipped_tokens"));
-    let skipped_buckets: Vec<f64> = (0..shard_info.speculate + 1).map(|x| x as f64).collect();
+    // let skipped_matcher = Matcher::Full(String::from("tgi_request_skipped_tokens"));
+    // let skipped_buckets: Vec<f64> = (0..shard_info.speculate + 1).map(|x| x as f64).collect();
 
     // Prometheus handler
     let builder = PrometheusBuilder::new()
@@ -1543,12 +1884,134 @@ pub async fn run(
         .set_buckets_for_metric(max_new_tokens_matcher, &max_new_tokens_buckets)
         .unwrap()
         .set_buckets_for_metric(batch_size_matcher, &batch_size_buckets)
-        .unwrap()
-        .set_buckets_for_metric(skipped_matcher, &skipped_buckets)
         .unwrap();
-    let prom_handle = builder
-        .install_recorder()
-        .expect("failed to install metrics recorder");
+    // .set_buckets_for_metric(skipped_matcher, &skipped_buckets)
+    // .unwrap();
+    // See: https://github.com/metrics-rs/metrics/issues/467#issuecomment-2022755151
+    let (recorder, _) = builder
+        .build()
+        .expect("failed to build prometheus recorder");
+    let prom_handle = recorder.handle();
+    metrics::set_global_recorder(recorder).expect("Failed to set global recorder");
+
+    // Metrics descriptions
+    metrics::describe_counter!("tgi_request_success", "Number of successful requests");
+    metrics::describe_histogram!(
+        "tgi_request_duration",
+        metrics::Unit::Seconds,
+        "Request duration"
+    );
+    metrics::describe_histogram!(
+        "tgi_request_validation_duration",
+        metrics::Unit::Seconds,
+        "Request validation duration"
+    );
+    metrics::describe_histogram!(
+        "tgi_request_queue_duration",
+        metrics::Unit::Seconds,
+        "Request queue duration"
+    );
+    metrics::describe_histogram!(
+        "tgi_request_inference_duration",
+        metrics::Unit::Seconds,
+        "Request inference duration"
+    );
+    metrics::describe_histogram!(
+        "tgi_request_mean_time_per_token_duration",
+        metrics::Unit::Seconds,
+        "Mean time per token per request"
+    );
+    metrics::describe_histogram!(
+        "tgi_request_generated_tokens",
+        metrics::Unit::Count,
+        "Generated tokens per request"
+    );
+    metrics::describe_counter!(
+        "tgi_batch_inference_count",
+        metrics::Unit::Count,
+        "Inference calls per method (prefill or decode)"
+    );
+    metrics::describe_counter!(
+        "tgi_request_count",
+        metrics::Unit::Count,
+        "Total number of requests"
+    );
+    metrics::describe_counter!(
+        "tgi_batch_inference_success",
+        metrics::Unit::Count,
+        "Number of successful inference calls per method (prefill or decode)"
+    );
+    metrics::describe_gauge!(
+        "tgi_batch_current_size",
+        metrics::Unit::Count,
+        "Current batch size"
+    );
+    metrics::describe_gauge!("tgi_queue_size", metrics::Unit::Count, "Current queue size");
+    metrics::describe_gauge!(
+        "tgi_batch_current_max_tokens",
+        metrics::Unit::Count,
+        "Maximum tokens for the current batch"
+    );
+    metrics::describe_gauge!(
+        "tgi_batch_total_tokens",
+        metrics::Unit::Count,
+        "Maximum amount of tokens in total."
+    );
+    metrics::describe_histogram!(
+        "tgi_request_max_new_tokens",
+        metrics::Unit::Count,
+        "Maximum new tokens per request"
+    );
+    metrics::describe_histogram!(
+        "tgi_batch_inference_duration",
+        metrics::Unit::Seconds,
+        "Batch inference duration"
+    );
+    metrics::describe_histogram!(
+        "tgi_batch_forward_duration",
+        metrics::Unit::Seconds,
+        "Batch forward duration per method (prefill or decode)"
+    );
+    metrics::describe_histogram!(
+        "tgi_request_skipped_tokens",
+        metrics::Unit::Count,
+        "Speculated tokens per request"
+    );
+    metrics::describe_histogram!(
+        "tgi_batch_filter_duration",
+        metrics::Unit::Seconds,
+        "Time spent filtering batches and sending generated tokens per method (prefill or decode)"
+    );
+    metrics::describe_histogram!(
+        "tgi_request_queue_duration",
+        metrics::Unit::Seconds,
+        "Time spent in the queue per request"
+    );
+    metrics::describe_histogram!(
+        "tgi_request_validation_duration",
+        metrics::Unit::Seconds,
+        "Time spent validating the request"
+    );
+    metrics::describe_histogram!(
+        "tgi_request_duration",
+        metrics::Unit::Seconds,
+        "Total time spent processing the request"
+    );
+    metrics::describe_histogram!(
+        "tgi_batch_decode_duration",
+        metrics::Unit::Seconds,
+        "Time spent decoding a batch per method (prefill or decode)"
+    );
+    metrics::describe_histogram!(
+        "tgi_request_input_length",
+        metrics::Unit::Count,
+        "Input token length per request"
+    );
+    metrics::describe_histogram!(
+        "tgi_batch_next_size",
+        metrics::Unit::Count,
+        "Batch size of the next batch"
+    );
 
     // CORS layer
     let allow_origin = allow_origin.unwrap_or(AllowOrigin::any());
@@ -1561,18 +2024,18 @@ pub async fn run(
     let info = Info {
         model_id: model_info.model_id,
         model_sha: model_info.sha,
-        model_dtype: shard_info.dtype,
-        model_device_type: shard_info.device_type,
+        // model_dtype: shard_info.dtype,
+        // model_device_type: shard_info.device_type,
         model_pipeline_tag: model_info.pipeline_tag,
         max_concurrent_requests,
         max_best_of,
         max_stop_sequences,
-        max_input_length,
+        max_input_tokens,
         max_total_tokens,
-        waiting_served_ratio,
-        max_batch_total_tokens,
-        max_waiting_tokens,
-        max_batch_size,
+        // waiting_served_ratio,
+        // max_batch_total_tokens,
+        // max_waiting_tokens,
+        // max_batch_size,
         validation_workers,
         max_client_batch_size,
         router: env!("CARGO_PKG_NAME"),
@@ -1581,46 +2044,105 @@ pub async fn run(
         docker_label: option_env!("DOCKER_LABEL"),
     };
 
-    // Define VertextApiDoc conditionally only if the "google" feature is enabled
-    let doc = {
-        // avoid `mut` if possible
-        #[cfg(feature = "google")]
-        {
-            use crate::VertexInstance;
+    #[allow(unused_mut)] // mut is needed for conditional compilation
+    let mut doc = ApiDoc::openapi();
 
-            #[derive(OpenApi)]
-            #[openapi(
-                paths(vertex_compatibility),
-                components(schemas(VertexInstance, VertexRequest, VertexResponse))
-            )]
-            struct VertextApiDoc;
+    #[cfg(feature = "google")]
+    {
+        use crate::vertex::__path_vertex_compatibility;
+        use crate::vertex::{VertexInstance, VertexRequest, VertexResponse};
 
-            // limiting mutability to the smallest scope necessary
-            let mut doc = ApiDoc::openapi();
-            doc.merge(VertextApiDoc::openapi());
-            doc
-        }
-        #[cfg(not(feature = "google"))]
-        ApiDoc::openapi()
-    };
+        #[derive(OpenApi)]
+        #[openapi(
+            paths(vertex_compatibility),
+            components(schemas(VertexInstance, VertexRequest, VertexResponse))
+        )]
+        struct VertexApiDoc;
+
+        doc.merge(VertexApiDoc::openapi());
+    }
+
+    #[cfg(feature = "kserve")]
+    {
+        use crate::kserve::{
+            InferenceOutput, InferenceRequest, LiveResponse, MetadataServerResponse, OutputChunk,
+            ReadyResponse,
+        };
+        use crate::kserve::{
+            __path_kerve_server_metadata, __path_kserve_health_live, __path_kserve_health_ready,
+            __path_kserve_model_infer, __path_kserve_model_metadata,
+            __path_kserve_model_metadata_ready,
+        };
+
+        #[derive(OpenApi)]
+        #[openapi(
+            paths(
+                kserve_health_live,
+                kserve_health_ready,
+                kerve_server_metadata,
+                kserve_model_metadata,
+                kserve_model_metadata_ready,
+                kserve_model_infer,
+            ),
+            components(schemas(
+                InferenceOutput,
+                InferenceRequest,
+                LiveResponse,
+                MetadataServerResponse,
+                OutputChunk,
+                ReadyResponse,
+            ))
+        )]
+        struct KServeApiDoc;
+
+        doc.merge(KServeApiDoc::openapi());
+    }
 
     // Configure Swagger UI
     let swagger_ui = SwaggerUi::new("/docs").url("/api-doc/openapi.json", doc);
 
     // Define base and health routes
-    let base_routes = Router::new()
+    let mut base_routes = Router::new()
         .route("/", post(compat_generate))
-        .route("/", get(health))
-        .route("/info", get(get_model_info))
         .route("/generate", post(generate))
         .route("/generate_stream", post(generate_stream))
         .route("/v1/chat/completions", post(chat_completions))
         .route("/v1/completions", post(completions))
         .route("/vertex", post(vertex_compatibility))
-        .route("/tokenize", post(tokenize))
+        .route("/tokenize", post(tokenize));
+
+    if let Some(api_key) = api_key {
+        let mut prefix = "Bearer ".to_string();
+        prefix.push_str(&api_key);
+
+        // Leak to allow FnMut
+        let api_key: &'static str = prefix.leak();
+
+        let auth = move |headers: HeaderMap,
+                         request: axum::extract::Request,
+                         next: axum::middleware::Next| async move {
+            match headers.get(AUTHORIZATION) {
+                Some(token) => match token.to_str() {
+                    Ok(token_str) if token_str.to_lowercase() == api_key.to_lowercase() => {
+                        let response = next.run(request).await;
+                        Ok(response)
+                    }
+                    _ => Err(StatusCode::UNAUTHORIZED),
+                },
+                None => Err(StatusCode::UNAUTHORIZED),
+            }
+        };
+
+        base_routes = base_routes.layer(axum::middleware::from_fn(auth))
+    }
+    let info_routes = Router::new()
+        .route("/", get(health))
+        .route("/chat_tokenize", post(get_chat_tokenize))
+        .route("/info", get(get_model_info))
         .route("/health", get(health))
         .route("/ping", get(health))
-        .route("/metrics", get(metrics));
+        .route("/metrics", get(metrics))
+        .route("/v1/models", get(openai_get_model_info));
 
     // Conditional AWS Sagemaker route
     let aws_sagemaker_route = if messages_api_enabled {
@@ -1636,6 +2158,7 @@ pub async fn run(
     let mut app = Router::new()
         .merge(swagger_ui)
         .merge(base_routes)
+        .merge(info_routes)
         .merge(aws_sagemaker_route);
 
     #[cfg(feature = "google")]
@@ -1652,10 +2175,30 @@ pub async fn run(
         }
     }
 
+    #[cfg(feature = "kserve")]
+    {
+        tracing::info!("Built with `kserve` feature");
+        app = app
+            .route(
+                "/v2/models/:model_name/versions/:model_version/infer",
+                post(kserve_model_infer),
+            )
+            .route(
+                "/v2/models/:model_name/versions/:model_version",
+                get(kserve_model_metadata),
+            )
+            .route("/v2/health/ready", get(kserve_health_ready))
+            .route("/v2/health/live", get(kserve_health_live))
+            .route("/v2", get(kerve_server_metadata))
+            .route(
+                "/v2/models/:model_name/versions/:model_version/ready",
+                get(kserve_model_metadata_ready),
+            );
+    }
+
     // add layers after routes
     app = app
         .layer(Extension(info))
-        .layer(Extension(health_ext.clone()))
         .layer(Extension(compat_return_full_text))
         .layer(Extension(infer))
         .layer(Extension(compute_type))
@@ -1663,49 +2206,14 @@ pub async fn run(
         .layer(OtelAxumLayer::default())
         .layer(cors_layer);
 
+    tracing::info!("Connected");
+
     if ngrok {
         #[cfg(feature = "ngrok")]
         {
-            use ngrok::config::TunnelBuilder;
-
-            let _ = addr;
-
-            let authtoken =
-                ngrok_authtoken.expect("`ngrok-authtoken` must be set when using ngrok tunneling");
-
-            let edge = ngrok_edge.expect("`ngrok-edge` must be set when using ngrok tunneling");
-
-            let tunnel = ngrok::Session::builder()
-                .authtoken(authtoken)
-                .connect()
-                .await
-                .unwrap()
-                .labeled_tunnel()
-                .label("edge", edge);
-
-            let listener = tunnel.listen().await.unwrap();
-
-            // Run prom metrics and health locally too
-            tokio::spawn(
-                axum::Server::bind(&addr)
-                    .serve(
-                        Router::new()
-                            .route("/health", get(health))
-                            .route("/metrics", get(metrics))
-                            .layer(Extension(health_ext))
-                            .layer(Extension(prom_handle))
-                            .into_make_service(),
-                    )
-                    //Wait until all requests are finished to shut down
-                    .with_graceful_shutdown(shutdown_signal()),
-            );
+            panic!("ngrok feature is not functional with axum=0.7 and hyper=1, waiting on https://github.com/ngrok/ngrok-rust/pull/137/files to re-enable.");
 
             // Run server
-            axum::Server::builder(listener)
-                .serve(app.into_make_service())
-                //Wait until all requests are finished to shut down
-                .with_graceful_shutdown(shutdown_signal())
-                .await?;
         }
         #[cfg(not(feature = "ngrok"))]
         {
@@ -1718,15 +2226,78 @@ pub async fn run(
         }
     } else {
         // Run server
-        axum::Server::bind(&addr)
-            .serve(app.into_make_service())
-            // Wait until all requests are finished to shut down
+
+        let listener = tokio::net::TcpListener::bind(&addr).await.unwrap();
+        axum::serve(listener, app)
             .with_graceful_shutdown(shutdown_signal())
-            .await?;
+            .await
+            .map_err(|err| WebServerError::Axum(Box::new(err)))?;
     }
     Ok(())
 }
 
+/// get model info from the Huggingface Hub
+pub async fn get_hub_model_info(api: &ApiRepo) -> Option<HubModelInfo> {
+    let response = api.info_request().send().await.ok()?;
+
+    if response.status().is_success() {
+        let hub_model_info: HubModelInfo =
+            serde_json::from_str(&response.text().await.ok()?).ok()?;
+        if let Some(sha) = &hub_model_info.sha {
+            tracing::info!(
+                "Serving revision {sha} of model {}",
+                hub_model_info.model_id
+            );
+        }
+        Some(hub_model_info)
+    } else {
+        None
+    }
+}
+
+/// get base tokenizer
+pub async fn get_base_tokenizer(api: &Api, api_repo: &ApiRepo) -> Option<PathBuf> {
+    let config_filename = api_repo.get("config.json").await.ok()?;
+
+    // Open the file in read-only mode with buffer.
+    let file = File::open(config_filename).ok()?;
+    let reader = BufReader::new(file);
+
+    // Read the JSON contents of the file as an instance of `User`.
+    let config: serde_json::Value = serde_json::from_reader(reader).ok()?;
+
+    if let Some(serde_json::Value::String(base_model_id)) = config.get("base_model_name_or_path") {
+        let api_base_repo = api.repo(Repo::with_revision(
+            base_model_id.to_string(),
+            RepoType::Model,
+            "main".to_string(),
+        ));
+
+        api_base_repo.get("tokenizer.json").await.ok()
+    } else {
+        None
+    }
+}
+
+/// get tokenizer_config from the Huggingface Hub
+pub async fn get_tokenizer_config(api_repo: &ApiRepo) -> Option<HubTokenizerConfig> {
+    let tokenizer_config_filename = api_repo.get("tokenizer_config.json").await.ok()?;
+
+    // Open the file in read-only mode with buffer.
+    let file = File::open(tokenizer_config_filename).ok()?;
+    let reader = BufReader::new(file);
+
+    // Read the JSON contents of the file as an instance of 'HubTokenizerConfig'.
+    let tokenizer_config: HubTokenizerConfig = serde_json::from_reader(reader)
+        .map_err(|e| {
+            tracing::warn!("Unable to parse tokenizer config: {}", e);
+            e
+        })
+        .ok()?;
+
+    Some(tokenizer_config)
+}
+
 /// Shutdown signal handler
 async fn shutdown_signal() {
     let ctrl_c = async {
@@ -1755,17 +2326,6 @@ async fn shutdown_signal() {
     opentelemetry::global::shutdown_tracer_provider();
 }
 
-impl From<i32> for FinishReason {
-    fn from(finish_reason: i32) -> Self {
-        let finish_reason = text_generation_client::FinishReason::try_from(finish_reason).unwrap();
-        match finish_reason {
-            text_generation_client::FinishReason::Length => FinishReason::Length,
-            text_generation_client::FinishReason::EosToken => FinishReason::EndOfSequenceToken,
-            text_generation_client::FinishReason::StopSequence => FinishReason::StopSequence,
-        }
-    }
-}
-
 /// Convert to Axum supported formats
 impl From<InferError> for (StatusCode, Json<ErrorResponse>) {
     fn from(err: InferError) -> Self {
@@ -1774,7 +2334,9 @@ impl From<InferError> for (StatusCode, Json<ErrorResponse>) {
             InferError::Overloaded(_) => StatusCode::TOO_MANY_REQUESTS,
             InferError::ValidationError(_) => StatusCode::UNPROCESSABLE_ENTITY,
             InferError::IncompleteGeneration => StatusCode::INTERNAL_SERVER_ERROR,
+            InferError::IncompleteGenerationStream => StatusCode::INTERNAL_SERVER_ERROR,
             InferError::TemplateError(_) => StatusCode::UNPROCESSABLE_ENTITY,
+            InferError::MissingTemplateVariable(_) => StatusCode::UNPROCESSABLE_ENTITY,
             InferError::ToolError(_) => StatusCode::UNPROCESSABLE_ENTITY,
         };
 
@@ -1798,3 +2360,163 @@ impl From<InferError> for Event {
             .unwrap()
     }
 }
+
+#[derive(Debug, Error)]
+pub enum WebServerError {
+    #[error("Axum error: {0}")]
+    Axum(#[from] axum::BoxError),
+}
+
+type PreparedInput = (String, Option<GrammarType>, bool);
+
+pub(crate) fn prepare_chat_input(
+    infer: &Infer,
+    response_format: Option<GrammarType>,
+    tools: Option<Vec<Tool>>,
+    tool_choice: ToolChoice,
+    tool_prompt: &str,
+    guideline: Option<String>,
+    messages: Vec<Message>,
+) -> Result<PreparedInput, InferError> {
+    if response_format.is_some() && tools.is_some() {
+        return Err(InferError::ToolError(
+            "Grammar and tools are mutually exclusive".into(),
+        ));
+    }
+
+    // when response_format is set, tools are not included when applying the chat template to generate inputs
+    if let Some(format) = response_format {
+        let inputs = infer.apply_chat_template(guideline, messages, None)?;
+        return Ok((inputs, Some(format), false));
+    }
+
+    // when no response_format is set and tools are included, apply the chat template with the tools
+    // to generate inputs
+    if let Some(tools) = tools {
+        let (updated_tools, tool_schema) = ToolGrammar::apply(tools, tool_choice)?;
+
+        let grammar = tool_schema
+            .as_ref()
+            .map(|t| GrammarType::Json(serde_json::json!(t)));
+
+        let inputs: String = infer.apply_chat_template(
+            guideline,
+            messages,
+            Some((updated_tools, tool_prompt.into())),
+        )?;
+        return Ok((inputs, grammar, tool_schema.is_some()));
+    }
+
+    // if no response_format or tools are set simply apply the chat template to generate inputs
+    let inputs = infer.apply_chat_template(guideline, messages, None)?;
+    Ok((inputs, None, false))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::ChatTemplateVersions;
+    use crate::HubTokenizerConfig;
+    use crate::TokenizerConfigToken;
+    use crate::Tool;
+
+    use serde_json::json;
+
+    #[test]
+    fn test_prepare_chat_input() {
+        // Mock Backend to avoid network requests
+        struct MockBackend;
+
+        impl Backend for MockBackend {
+            fn schedule(
+                &self,
+                _request: crate::validation::ValidGenerateRequest,
+            ) -> Result<
+                tokio_stream::wrappers::UnboundedReceiverStream<
+                    Result<InferStreamResponse, InferError>,
+                >,
+                InferError,
+            > {
+                unimplemented!("Never called in this test");
+            }
+            fn health<'a, 'async_trait>(
+                &'a self,
+                _current_health: bool,
+            ) -> core::pin::Pin<
+                Box<dyn core::future::Future<Output = bool> + core::marker::Send + 'async_trait>,
+            >
+            where
+                'a: 'async_trait,
+                Self: 'async_trait,
+            {
+                unimplemented!("Never called in this test");
+            }
+        }
+
+        let backend = MockBackend {};
+
+        let mut tokenizer_config = HubTokenizerConfig::default();
+
+        // mock tokenizer config values
+        tokenizer_config.bos_token = Some(TokenizerConfigToken::String("<s>".to_string()));
+        tokenizer_config.eos_token = Some(TokenizerConfigToken::String("</s>".to_string()));
+        tokenizer_config.chat_template = Some(
+            ChatTemplateVersions::Single("{%- if messages[0][\"role\"] == \"system\" %}\n    {%- set system_message = messages[0][\"content\"] %}\n    {%- set loop_messages = messages[1:] %}\n{%- else %}\n    {%- set loop_messages = messages %}\n{%- endif %}\n{%- if not tools is defined %}\n    {%- set tools = none %}\n{%- endif %}\n{%- set user_messages = loop_messages | selectattr(\"role\", \"equalto\", \"user\") | list %}\n\n{#- This block checks for alternating user/assistant messages, skipping tool calling messages #}\n{%- set ns = namespace() %}\n{%- set ns.index = 0 %}\n{%- for message in loop_messages %}\n    {%- if not (message.role == \"tool\" or message.role == \"tool_results\" or (message.tool_calls is defined and message.tool_calls is not none)) %}\n        {%- if (message[\"role\"] == \"user\") != (ns.index % 2 == 0) %}\n            {{- raise_exception(\"After the optional system message, conversation roles must alternate user/assistant/user/assistant/...\") }}\n        {%- endif %}\n        {%- set ns.index = ns.index + 1 %}\n    {%- endif %}\n{%- endfor %}\n\n{{- bos_token }}\n{%- for message in loop_messages %}\n    {%- if message[\"role\"] == \"user\" %}\n        {%- if tools is not none and (message == user_messages[-1]) %}\n            {{- \"[AVAILABLE_TOOLS] [\" }}\n            {%- for tool in tools %}\n                {%- set tool = tool.function %}\n                {{- '{\"type\": \"function\", \"function\": {' }}\n                {%- for key, val in tool.items() if key != \"return\" %}\n                    {%- if val is string %}\n                        {{- '\"' + key + '\": \"' + val + '\"' }}\n                    {%- else %}\n                        {{- '\"' + key + '\": ' + val|tojson }}\n                    {%- endif %}\n                    {%- if not loop.last %}\n                        {{- \", \" }}\n                    {%- endif %}\n                {%- endfor %}\n                {{- \"}}\" }}\n                {%- if not loop.last %}\n                    {{- \", \" }}\n                {%- else %}\n                    {{- \"]\" }}\n                {%- endif %}\n            {%- endfor %}\n            {{- \"[/AVAILABLE_TOOLS]\" }}\n            {%- endif %}\n        {%- if loop.last and system_message is defined %}\n            {{- \"[INST] \" + system_message + \"\\n\\n\" + message[\"content\"] + \"[/INST]\" }}\n        {%- else %}\n            {{- \"[INST] \" + message[\"content\"] + \"[/INST]\" }}\n        {%- endif %}\n    {%- elif message.tool_calls is defined and message.tool_calls is not none %}\n        {{- \"[TOOL_CALLS] [\" }}\n        {%- for tool_call in message.tool_calls %}\n            {%- set out = tool_call.function|tojson %}\n            {{- out[:-1] }}\n            {%- if not tool_call.id is defined or tool_call.id|length != 9 %}\n                {{- raise_exception(\"Tool call IDs should be alphanumeric strings with length 9!\") }}\n            {%- endif %}\n            {{- ', \"id\": \"' + tool_call.id + '\"}' }}\n            {%- if not loop.last %}\n                {{- \", \" }}\n            {%- else %}\n                {{- \"]\" + eos_token }}\n            {%- endif %}\n        {%- endfor %}\n    {%- elif message[\"role\"] == \"assistant\" %}\n        {{- \" \" + message[\"content\"]|trim + eos_token}}\n    {%- elif message[\"role\"] == \"tool_results\" or message[\"role\"] == \"tool\" %}\n        {%- if message.content is defined and message.content.content is defined %}\n            {%- set content = message.content.content %}\n        {%- else %}\n            {%- set content = message.content %}\n        {%- endif %}\n        {{- '[TOOL_RESULTS] {\"content\": ' + content|string + \", \" }}\n        {%- if not message.tool_call_id is defined or message.tool_call_id|length != 9 %}\n            {{- raise_exception(\"Tool call IDs should be alphanumeric strings with length 9!\") }}\n        {%- endif %}\n        {{- '\"call_id\": \"' + message.tool_call_id + '\"}[/TOOL_RESULTS]' }}\n    {%- else %}\n        {{- raise_exception(\"Only user and assistant roles are supported, with the exception of an initial optional system message!\") }}\n    {%- endif %}\n{%- endfor %}\n".to_string())
+        );
+
+        let infer = Infer::new(
+            backend,
+            Validation::new(1, None, None, None, 1, 1, 1, 1, 1, false),
+            1,
+            tokenizer_config,
+            HubProcessorConfig::default(),
+        );
+        let response_format = None;
+        let tools = Some(vec![Tool {
+            r#type: "function".to_string(),
+            function: FunctionDefinition {
+                name: "get_current_weather".to_string(),
+                description: Some("Get the current weather".to_string()),
+                arguments: json!({
+                    "type": "object",
+                    "properties": {
+                        "location": {
+                            "type": "string",
+                            "description": "The city and state, e.g. San Francisco, CA"
+                        },
+                        "format": {
+                            "type": "string",
+                            "enum": ["celsius", "fahrenheit"],
+                            "description": "The temperature unit to use. Infer this from the users location."
+                        }
+                    },
+                    "required": ["location", "format"]
+                }),
+            },
+        }]);
+        let tool_prompt = "Given the functions available, please respond with a JSON for a function call with its proper arguments that best answers the given prompt. Respond in the format {name: function name, parameters: dictionary of argument name and its value}.Do not use variables.";
+        let guideline = None;
+        let messages = vec![Message {
+            name: None,
+            role: "user".to_string(),
+            content: MessageContent::SingleText(
+                "What is the weather like in New York?".to_string(),
+            ),
+        }];
+
+        let result = prepare_chat_input(
+            &infer,
+            response_format,
+            tools,
+            ToolChoice(None),
+            tool_prompt,
+            guideline,
+            messages,
+        );
+
+        assert!(result.is_ok());
+        let (inputs, _grammar, using_tools) = result.unwrap();
+        assert_eq!(using_tools, true);
+        assert_eq!(inputs, "<s>[AVAILABLE_TOOLS] [{\"type\": \"function\", \"function\": {\"arguments\": {\"properties\":{\"format\":{\"description\":\"The temperature unit to use. Infer this from the users location.\",\"enum\":[\"celsius\",\"fahrenheit\"],\"type\":\"string\"},\"location\":{\"description\":\"The city and state, e.g. San Francisco, CA\",\"type\":\"string\"}},\"required\":[\"location\",\"format\"],\"type\":\"object\"}, \"description\": \"Get the current weather\", \"name\": \"get_current_weather\"}}, {\"type\": \"function\", \"function\": {\"arguments\": {\"properties\":{\"error\":{\"description\":\"The error or issue to notify\",\"type\":\"string\"}},\"required\":[\"error\"],\"type\":\"object\"}, \"description\": \"Notify an error or issue\", \"name\": \"notify_error\"}}][/AVAILABLE_TOOLS][INST] What is the weather like in New York?\n---\nGiven the functions available, please respond with a JSON for a function call with its proper arguments that best answers the given prompt. Respond in the format {name: function name, parameters: dictionary of argument name and its value}.Do not use variables.[/INST]".to_string());
+    }
+}
diff --git a/router/src/usage_stats.rs b/router/src/usage_stats.rs
new file mode 100644
index 00000000..0282ac63
--- /dev/null
+++ b/router/src/usage_stats.rs
@@ -0,0 +1,360 @@
+use crate::config::Config;
+use clap::ValueEnum;
+use csv::ReaderBuilder;
+use reqwest::header::HeaderMap;
+use serde::Serialize;
+use std::{
+    fs::File,
+    io::{self, BufRead},
+    path::Path,
+    process::Command,
+    time::Duration,
+};
+use uuid::Uuid;
+
+const TELEMETRY_URL: &str = "https://huggingface.co/api/telemetry/tgi";
+
+#[derive(Copy, Clone, Debug, Serialize, ValueEnum)]
+pub enum UsageStatsLevel {
+    On,
+    NoStack,
+    Off,
+}
+
+#[derive(Debug, Clone, Serialize)]
+pub struct UserAgent {
+    pub uid: String,
+    pub args: Args,
+    pub env: Env,
+}
+
+impl UserAgent {
+    pub fn new(reduced_args: Args) -> Self {
+        Self {
+            uid: Uuid::new_v4().to_string(),
+            args: reduced_args,
+            env: Env::new(),
+        }
+    }
+}
+
+#[derive(Serialize, Debug)]
+pub enum EventType {
+    Start,
+    Stop,
+    Error,
+}
+
+#[derive(Debug, Serialize)]
+pub struct UsageStatsEvent {
+    user_agent: UserAgent,
+    event_type: EventType,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    error_reason: Option<String>,
+}
+
+impl UsageStatsEvent {
+    pub fn new(user_agent: UserAgent, event_type: EventType, error_reason: Option<String>) -> Self {
+        Self {
+            user_agent,
+            event_type,
+            error_reason,
+        }
+    }
+    pub async fn send(&self) {
+        let mut headers = HeaderMap::new();
+        headers.insert("Content-Type", "application/json".parse().unwrap());
+        let body = serde_json::to_string(&self).unwrap();
+        let client = reqwest::Client::new();
+        let _ = client
+            .post(TELEMETRY_URL)
+            .headers(headers)
+            .body(body)
+            .timeout(Duration::from_secs(5))
+            .send()
+            .await;
+    }
+}
+
+#[derive(Debug, Clone, Serialize)]
+pub struct Args {
+    model_config: Option<Config>,
+    tokenizer_class: Option<String>,
+    max_concurrent_requests: usize,
+    max_best_of: usize,
+    max_stop_sequences: usize,
+    max_top_n_tokens: u32,
+    max_input_tokens: usize,
+    max_total_tokens: usize,
+    // waiting_served_ratio: f32,
+    // max_batch_prefill_tokens: u32,
+    // max_batch_total_tokens: Option<u32>,
+    // max_waiting_tokens: usize,
+    // max_batch_size: Option<usize>,
+    revision: Option<String>,
+    validation_workers: usize,
+    messages_api_enabled: bool,
+    disable_grammar_support: bool,
+    max_client_batch_size: usize,
+    usage_stats_level: UsageStatsLevel,
+}
+
+impl Args {
+    #[allow(clippy::too_many_arguments)]
+    pub fn new(
+        model_config: Option<Config>,
+        tokenizer_class: Option<String>,
+        max_concurrent_requests: usize,
+        max_best_of: usize,
+        max_stop_sequences: usize,
+        max_top_n_tokens: u32,
+        max_input_tokens: usize,
+        max_total_tokens: usize,
+        // waiting_served_ratio: f32,
+        // max_batch_prefill_tokens: u32,
+        // max_batch_total_tokens: Option<u32>,
+        // max_waiting_tokens: usize,
+        // max_batch_size: Option<usize>,
+        revision: Option<String>,
+        validation_workers: usize,
+        messages_api_enabled: bool,
+        disable_grammar_support: bool,
+        max_client_batch_size: usize,
+        usage_stats_level: UsageStatsLevel,
+    ) -> Self {
+        Self {
+            model_config,
+            tokenizer_class,
+            max_concurrent_requests,
+            max_best_of,
+            max_stop_sequences,
+            max_top_n_tokens,
+            max_input_tokens,
+            max_total_tokens,
+            // waiting_served_ratio,
+            // max_batch_prefill_tokens,
+            // max_batch_total_tokens,
+            // max_waiting_tokens,
+            // max_batch_size,
+            revision,
+            validation_workers,
+            messages_api_enabled,
+            disable_grammar_support,
+            max_client_batch_size,
+            usage_stats_level,
+        }
+    }
+}
+
+/// This is more or less a copy of the code from the `text-generation-launcher` crate to avoid a dependency
+#[derive(Serialize, Debug, Clone)]
+pub struct Env {
+    git_sha: &'static str,
+    docker_label: &'static str,
+    nvidia_info: Option<Vec<NvidiaSmiInfo>>,
+    xpu_info: Option<Vec<XpuSmiInfo>>,
+    system_env: SystemInfo,
+}
+
+#[derive(Debug, Serialize, Clone)]
+struct NvidiaSmiInfo {
+    name: String,
+    pci_bus_id: String,
+    driver_version: String,
+    pstate: String,
+    pcie_link_gen_max: String,
+    pcie_link_gen_current: String,
+    temperature_gpu: String,
+    utilization_gpu: String,
+    utilization_memory: String,
+    memory_total: String,
+    memory_free: String,
+    memory_used: String,
+    reset_status_reset_required: String,
+    reset_status_drain_and_reset_recommended: String,
+    compute_cap: String,
+    ecc_errors_corrected_volatile_total: String,
+    mig_mode_current: String,
+    power_draw_instant: String,
+    power_limit: String,
+}
+
+impl NvidiaSmiInfo {
+    fn new() -> Option<Vec<NvidiaSmiInfo>> {
+        let output = Command::new("nvidia-smi")
+            .args([
+                "--query-gpu=name,pci.bus_id,driver_version,pstate,pcie.link.gen.max,pcie.link.gen.gpucurrent,temperature.gpu,utilization.gpu,utilization.memory,memory.total,memory.free,memory.used,reset_status.reset_required,reset_status.drain_and_reset_recommended,compute_cap,ecc.errors.corrected.volatile.total,mig.mode.current,power.draw.instant,power.limit",
+                "--format=csv"
+            ])
+            .output()
+            .ok()?;
+
+        if !output.status.success() {
+            return None;
+        }
+
+        let stdout = String::from_utf8(output.stdout).ok()?;
+
+        let mut rdr = ReaderBuilder::new()
+            .has_headers(true)
+            .from_reader(stdout.as_bytes());
+
+        let mut infos = Vec::new();
+
+        for result in rdr.records() {
+            let record = result.ok()?;
+            infos.push(NvidiaSmiInfo {
+                name: record[0].to_string(),
+                pci_bus_id: record[1].to_string(),
+                driver_version: record[2].to_string(),
+                pstate: record[3].to_string(),
+                pcie_link_gen_max: record[4].to_string(),
+                pcie_link_gen_current: record[5].to_string(),
+                temperature_gpu: record[6].to_string(),
+                utilization_gpu: record[7].to_string(),
+                utilization_memory: record[8].to_string(),
+                memory_total: record[9].to_string(),
+                memory_free: record[10].to_string(),
+                memory_used: record[11].to_string(),
+                reset_status_reset_required: record[12].to_string(),
+                reset_status_drain_and_reset_recommended: record[13].to_string(),
+                compute_cap: record[14].to_string(),
+                ecc_errors_corrected_volatile_total: record[15].to_string(),
+                mig_mode_current: record[16].to_string(),
+                power_draw_instant: record[17].to_string(),
+                power_limit: record[18].to_string(),
+            });
+        }
+
+        Some(infos)
+    }
+}
+
+#[derive(Debug, Serialize, Clone)]
+struct XpuSmiInfo {
+    device_id: usize,
+    gpu_utilization: f32,
+    gpu_power: f32,
+    gpu_core_temperature: f32,
+    gpu_memory_bandwidth_utilization: f32,
+}
+
+impl XpuSmiInfo {
+    /// based on this https://github.com/intel/xpumanager/blob/master/doc/smi_user_guide.md#dump-the-device-statistics-in-csv-format
+    fn new() -> Option<Vec<XpuSmiInfo>> {
+        let output = Command::new("xpu-smi")
+            .args([
+                "dump", "-d", "-1", "-m",
+                "0,1,3,17", // Metrics IDs: GPU Utilization, GPU Power, GPU Core Temperature, GPU Memory Bandwidth Utilization
+                "-n", "1", "-j",
+            ])
+            .output()
+            .ok()?;
+
+        if !output.status.success() {
+            return None;
+        }
+
+        let stdout = String::from_utf8(output.stdout).ok()?;
+        let mut infos = Vec::new();
+
+        let json_data: serde_json::Value = match serde_json::from_str(&stdout) {
+            Ok(data) => data,
+            Err(_) => return None,
+        };
+
+        if let Some(metrics_data) = json_data.as_array() {
+            for entry in metrics_data {
+                let device_id = entry["deviceId"].as_u64()? as usize;
+                let gpu_utilization = entry["metrics"][0].as_f64()? as f32;
+                let gpu_power = entry["metrics"][1].as_f64()? as f32;
+                let gpu_core_temperature = entry["metrics"][2].as_f64()? as f32;
+                let gpu_memory_bandwidth_utilization = entry["metrics"][3].as_f64()? as f32;
+
+                infos.push(XpuSmiInfo {
+                    device_id,
+                    gpu_utilization,
+                    gpu_power,
+                    gpu_core_temperature,
+                    gpu_memory_bandwidth_utilization,
+                });
+            }
+        }
+
+        Some(infos)
+    }
+}
+
+#[derive(Serialize, Debug, Clone)]
+pub struct SystemInfo {
+    cpu_count: usize,
+    cpu_type: String,
+    total_memory: u64,
+    architecture: String,
+    platform: String,
+}
+
+impl SystemInfo {
+    fn new() -> Self {
+        let mut system = sysinfo::System::new_all();
+        system.refresh_all();
+
+        let cpu_count = system.cpus().len();
+        let cpu_type = system.cpus()[0].brand().to_string();
+        let total_memory = system.total_memory();
+        let architecture = std::env::consts::ARCH.to_string();
+        let platform = format!(
+            "{}-{}-{}",
+            std::env::consts::OS,
+            std::env::consts::FAMILY,
+            std::env::consts::ARCH
+        );
+        Self {
+            cpu_count,
+            cpu_type,
+            total_memory,
+            architecture,
+            platform,
+        }
+    }
+}
+
+impl Default for Env {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl Env {
+    pub fn new() -> Self {
+        Self {
+            system_env: SystemInfo::new(),
+            nvidia_info: NvidiaSmiInfo::new(),
+            xpu_info: XpuSmiInfo::new(),
+            git_sha: option_env!("VERGEN_GIT_SHA").unwrap_or("N/A"),
+            docker_label: option_env!("DOCKER_LABEL").unwrap_or("N/A"),
+        }
+    }
+}
+
+pub fn is_container() -> io::Result<bool> {
+    let path = Path::new("/proc/self/cgroup");
+    let file = File::open(path)?;
+    let reader = io::BufReader::new(file);
+
+    for line in reader.lines() {
+        let line = line?;
+        // Check for common container runtimes
+        if line.contains("/docker/")
+            || line.contains("/docker-")
+            || line.contains("/kubepods/")
+            || line.contains("/kubepods-")
+            || line.contains("containerd")
+            || line.contains("crio")
+            || line.contains("podman")
+        {
+            return Ok(true);
+        }
+    }
+    Ok(false)
+}
diff --git a/router/src/validation.rs b/router/src/validation.rs
index ee48f705..85b4220b 100644
--- a/router/src/validation.rs
+++ b/router/src/validation.rs
@@ -1,22 +1,19 @@
-/// Copyright (C) 2024 Habana Labs, Ltd. an Intel Company.
-
-use crate::config::Config;
 /// Payload validation logic
+use crate::config::Config;
 use crate::validation::ValidationError::{BestOfSampling, BestOfSeed, EmptyInput};
-use crate::{GenerateParameters, GenerateRequest, GrammarType};
+use crate::{
+    GenerateParameters, GenerateRequest, GrammarType, HubPreprocessorConfig, Idefics2Preprocessor,
+};
+use base64::{engine::general_purpose::STANDARD, Engine};
+use image::{ImageFormat, ImageReader};
 use jsonschema::{Draft, JSONSchema};
 use rand::{thread_rng, Rng};
-use std::{cmp, env};
 use serde_json::Value;
 use std::io::Cursor;
-use text_generation_client::{
-    GrammarType as ProtoGrammarType, NextTokenChooserParameters, StoppingCriteriaParameters,
-};
+use std::iter;
+use std::sync::Arc;
 use thiserror::Error;
 use tokenizers::tokenizer::Tokenizer;
-// use tokenizers::TruncationDirection;
-use base64::{engine::general_purpose::STANDARD, Engine};
-use image::{io::Reader as ImageReader, ImageFormat};
 use tokio::sync::mpsc;
 use tokio::sync::oneshot;
 use tracing::{instrument, Span};
@@ -34,7 +31,6 @@ pub struct Validation {
     disable_grammar_support: bool,
     /// Channel to communicate with the background tokenization task
     sender: Option<mpsc::UnboundedSender<TokenizerRequest>>,
-    skip_tokenizer_in_tgi: bool,
 }
 
 impl Validation {
@@ -43,6 +39,7 @@ impl Validation {
         workers: usize,
         tokenizer: Option<Tokenizer>,
         config: Option<Config>,
+        preprocessor_config: Option<HubPreprocessorConfig>,
         max_best_of: usize,
         max_stop_sequences: usize,
         max_top_n_tokens: u32,
@@ -60,12 +57,18 @@ impl Validation {
             for _ in 0..workers {
                 let tokenizer_clone = tokenizer.clone();
                 let config_clone = config.clone();
+                let preprocessor_config_clone = preprocessor_config.clone();
                 let (tokenizer_sender, tokenizer_receiver) = mpsc::unbounded_channel();
                 senders.push(tokenizer_sender);
 
                 // Spawn worker
                 tokio::task::spawn_blocking(move || {
-                    tokenizer_worker(tokenizer_clone, config_clone, tokenizer_receiver)
+                    tokenizer_worker(
+                        tokenizer_clone,
+                        config_clone,
+                        preprocessor_config_clone,
+                        tokenizer_receiver,
+                    )
                 });
             }
 
@@ -77,10 +80,6 @@ impl Validation {
             None
         };
 
-        let skip_tokenizer_in_tgi = env::var("SKIP_TOKENIZER_IN_TGI")
-            .ok()
-            .map_or(false, |value| value.to_lowercase() == "true");
-
         Self {
             max_best_of,
             sender,
@@ -89,7 +88,6 @@ impl Validation {
             max_input_length,
             max_total_tokens,
             disable_grammar_support,
-            skip_tokenizer_in_tgi,
         }
     }
 
@@ -97,8 +95,9 @@ impl Validation {
     pub async fn tokenize(
         &self,
         inputs: String,
+        add_special_tokens: bool,
         truncate: Option<usize>,
-    ) -> Result<Option<(tokenizers::Encoding, String)>, ValidationError> {
+    ) -> Result<Option<(tokenizers::Encoding, Vec<Chunk>)>, ValidationError> {
         // If we have a fast tokenizer
         if let Some(sender) = &self.sender {
             // Create response channel
@@ -106,7 +105,11 @@ impl Validation {
             // Send request to the background validation task
             // Unwrap is safe here
             sender
-                .send(((inputs, truncate), response_sender, Span::current()))
+                .send((
+                    (inputs, add_special_tokens, truncate),
+                    response_sender,
+                    Span::current(),
+                ))
                 .unwrap();
 
             // Await on response channel
@@ -118,23 +121,25 @@ impl Validation {
         }
     }
 
+    #[allow(clippy::type_complexity)]
     #[instrument(skip(self, inputs))]
     async fn validate_input(
         &self,
         inputs: String,
+        add_special_tokens: bool,
         truncate: Option<usize>,
         max_new_tokens: Option<u32>,
-    ) -> Result<(String, usize, u32), ValidationError> {
+    ) -> Result<(Vec<Chunk>, Option<Vec<u32>>, usize, u32), ValidationError> {
         // If we have a fast tokenizer
-        if let Some((encoding, inputs)) = self.tokenize(inputs.clone(), truncate).await? {
+        if let Some((encoding, inputs)) = self
+            .tokenize(inputs.clone(), add_special_tokens, truncate)
+            .await?
+        {
             // Create response channel
-            let input_length = if self.skip_tokenizer_in_tgi {
-                inputs.chars().filter(|&c| c == ',').count() + 1
+            let input_length = if let Some(truncate) = truncate {
+                std::cmp::min(encoding.len(), truncate)
             } else {
-                cmp::max(
-                    encoding.len(),
-                    truncate.unwrap_or(self.max_input_length)
-                )
+                encoding.len()
             };
 
             // Get total tokens
@@ -162,8 +167,11 @@ impl Validation {
                 ));
             }
 
-            metrics::histogram!("tgi_request_input_length", input_length as f64);
-            Ok((inputs, input_length, max_new_tokens))
+            let ids = encoding.get_ids();
+            let input_ids = ids[ids.len().saturating_sub(input_length)..].to_owned();
+
+            metrics::histogram!("tgi_request_input_length").record(input_length as f64);
+            Ok((inputs, Some(input_ids), input_length, max_new_tokens))
         }
         // Return inputs without validation
         else {
@@ -177,21 +185,21 @@ impl Validation {
             } else {
                 return Err(ValidationError::UnsetMaxNewTokens);
             };
-            let input_length = if self.skip_tokenizer_in_tgi {
-                inputs.chars().filter(|&c| c == ',').count() + 1
-            } else {
-                truncate.unwrap_or(self.max_input_length)
-            };
+            let mut input_length = truncate.unwrap_or(self.max_input_length);
 
+            // We don't have a tokenizer, therefore we have no idea how long is the query, let
+            // them through and hope for the best.
             // Validate MaxNewTokens
             if (input_length as u32 + max_new_tokens) > self.max_total_tokens as u32 {
-                return Err(ValidationError::MaxNewTokens(
-                    self.max_total_tokens - self.max_input_length,
-                    max_new_tokens,
-                ));
+                input_length = input_length.saturating_sub(max_new_tokens as usize);
             }
 
-            Ok((inputs, input_length, max_new_tokens))
+            Ok((
+                vec![Chunk::Text(inputs)],
+                None,
+                input_length,
+                max_new_tokens,
+            ))
         }
     }
 
@@ -218,6 +226,7 @@ impl Validation {
             decoder_input_details,
             top_n_tokens,
             grammar,
+            adapter_id,
             ..
         } = request.parameters;
 
@@ -248,14 +257,6 @@ impl Validation {
             return Err(ValidationError::FrequencyPenalty);
         }
 
-        // TODO: enable watermark with fp8 quantization
-        let quantization_enabled = env::var("QUANT_CONFIG")
-            .ok()
-            .map_or(false, |value| !value.is_empty());
-        if watermark && quantization_enabled {
-            return Err(ValidationError::WatermarkWithQuantization);
-        }
-
         // Different because the proto default value is not a valid value
         // for the user
         let top_p = top_p
@@ -332,8 +333,13 @@ impl Validation {
             .unwrap_or(Ok(None))?;
 
         // Validate inputs
-        let (inputs, input_length, max_new_tokens) = self
-            .validate_input(request.inputs, truncate, max_new_tokens)
+        let (inputs, input_ids, input_length, max_new_tokens) = self
+            .validate_input(
+                request.inputs,
+                request.add_special_tokens,
+                truncate,
+                max_new_tokens,
+            )
             .await?;
 
         // TODO: we should build the FSM here and pass the compiled FSM instead of the grammar
@@ -343,13 +349,13 @@ impl Validation {
         // compiler and use that to build the FSM here.
 
         // Validate grammar and unpack the grammar and type for the proto message
-        let (grammar, grammar_type) = match grammar {
+        let grammar = match grammar {
             Some(grammar) => {
                 // Ensure that grammar is not set if it's not supported
                 if self.disable_grammar_support {
                     return Err(ValidationError::Grammar);
                 }
-                match grammar {
+                let valid_grammar = match grammar {
                     GrammarType::Json(json) => {
                         let json = match json {
                             // if value is a string, we need to parse it again to make sure its
@@ -366,20 +372,28 @@ impl Validation {
                             .compile(&json)
                             .map_err(|e| ValidationError::InvalidGrammar(e.to_string()))?;
 
-                        (
-                            // Serialize json to string
+                        // The schema can be valid but lack properties.
+                        // We need properties for the grammar to be successfully parsed in Python.
+                        // Therefore, we must check and throw an error if properties are missing.
+                        json.get("properties")
+                            .ok_or(ValidationError::InvalidGrammar(
+                                "Grammar must have a 'properties' field".to_string(),
+                            ))?;
+
+                        // Serialize json to string
+                        ValidGrammar::Json(
                             serde_json::to_string(&json)
                                 .map_err(|e| ValidationError::InvalidGrammar(e.to_string()))?,
-                            ProtoGrammarType::Json.into(),
                         )
                     }
-                    GrammarType::Regex(regex) => (regex, ProtoGrammarType::Regex.into()),
-                }
+                    GrammarType::Regex(regex) => ValidGrammar::Regex(regex),
+                };
+                Some(valid_grammar)
             }
-            None => (String::new(), ProtoGrammarType::None.into()),
+            None => None,
         };
 
-        let parameters = NextTokenChooserParameters {
+        let parameters = ValidParameters {
             temperature,
             repetition_penalty,
             frequency_penalty,
@@ -390,24 +404,26 @@ impl Validation {
             seed,
             watermark,
             grammar,
-            grammar_type,
         };
-        let stopping_parameters = StoppingCriteriaParameters {
+        let stopping_parameters = ValidStoppingParameters {
             max_new_tokens,
             stop_sequences,
             ignore_eos_token: false,
         };
 
-        metrics::histogram!("tgi_request_max_new_tokens", max_new_tokens as f64);
+        metrics::histogram!("tgi_request_max_new_tokens").record(max_new_tokens as f64);
 
         Ok(ValidGenerateRequest {
             inputs,
+            input_ids: input_ids.map(Arc::new),
+            add_special_tokens: request.add_special_tokens,
             decoder_input_details,
             input_length: input_length as u32,
             truncate: truncate.unwrap_or(self.max_input_length) as u32,
             parameters,
             stopping_parameters,
             top_n_tokens,
+            adapter_id,
         })
     }
 
@@ -445,13 +461,23 @@ async fn round_robin_task(
 fn tokenizer_worker(
     tokenizer: Tokenizer,
     config: Option<Config>,
+    preprocessor_config: Option<HubPreprocessorConfig>,
     mut receiver: mpsc::UnboundedReceiver<TokenizerRequest>,
 ) {
     // Loop over requests
-    while let Some(((inputs, truncate), response_tx, parent_span)) = receiver.blocking_recv() {
+    while let Some(((inputs, add_special_tokens, truncate), response_tx, parent_span)) =
+        receiver.blocking_recv()
+    {
         parent_span.in_scope(|| {
             response_tx
-                .send(prepare_input(inputs, truncate, &tokenizer, &config))
+                .send(prepare_input(
+                    inputs,
+                    truncate,
+                    add_special_tokens,
+                    &tokenizer,
+                    config.as_ref(),
+                    preprocessor_config.as_ref(),
+                ))
                 .unwrap_or(())
         })
     }
@@ -474,6 +500,7 @@ fn format_from_mimetype(mimetype: &str) -> Option<ImageFormat> {
         _ => None,
     }
 }
+
 fn format_to_mimetype(format: ImageFormat) -> String {
     match format {
         ImageFormat::Png => "image/png",
@@ -486,7 +513,7 @@ fn format_to_mimetype(format: ImageFormat) -> String {
     .to_string()
 }
 
-fn fetch_image(input: &str) -> Result<(String, usize, usize), ValidationError> {
+fn fetch_image(input: &str) -> Result<(Vec<u8>, String, usize, usize), ValidationError> {
     if input.starts_with("![](http://") || input.starts_with("![](https://") {
         let url = &input["![](".len()..input.len() - 1];
         let data = reqwest::blocking::get(url)?.bytes()?;
@@ -497,9 +524,7 @@ fn fetch_image(input: &str) -> Result<(String, usize, usize), ValidationError> {
         let height: usize = img.height().try_into()?;
         let width: usize = img.width().try_into()?;
         let mimetype = format_to_mimetype(format);
-        let encoded = STANDARD.encode(data);
-        let data_uri = format!("![](data:{mimetype};base64,{encoded})");
-        Ok((data_uri, height, width))
+        Ok((data.to_vec(), mimetype, height, width))
     } else if input.starts_with("![](data:") {
         // Remove ![](....)
         let content = &input["![](data:".len()..input.len() - 1];
@@ -516,9 +541,9 @@ fn fetch_image(input: &str) -> Result<(String, usize, usize), ValidationError> {
 
         let data = STANDARD.decode(content["base64,".len()..].as_bytes())?;
         let img = if let Some(format) = format_from_mimetype(mimetype) {
-            ImageReader::with_format(Cursor::new(data), format).decode()?
+            ImageReader::with_format(Cursor::new(&data), format).decode()?
         } else {
-            ImageReader::new(Cursor::new(data))
+            ImageReader::new(Cursor::new(&data))
                 .with_guessed_format()
                 .map_err(|_io_error| ValidationError::InvalidImageContent(content.to_string()))?
                 .decode()?
@@ -526,146 +551,202 @@ fn fetch_image(input: &str) -> Result<(String, usize, usize), ValidationError> {
 
         let height: usize = img.height().try_into()?;
         let width: usize = img.width().try_into()?;
-        Ok((input.to_string(), height, width))
+        Ok((data, mimetype.to_string(), height, width))
     } else {
         Err(ValidationError::InvalidImageContent(input.to_string()))
     }
 }
 
+fn image_tokens(
+    config: &Config,
+    preprocessor_config: Option<&HubPreprocessorConfig>,
+    height: usize,
+    width: usize,
+) -> String {
+    use Config::*;
+    use HubPreprocessorConfig::*;
+    match config {
+        Idefics => "<image>".to_string(),
+        Mllama => "<|image|>".to_string(),
+        Idefics2(config) => {
+            const FAKE: &str = "<fake_token_around_image>";
+            const IMAGE: &str = "<image>";
+
+            let slots = config.get_number_of_features(height, width);
+
+            let mut image_string = String::with_capacity(2 * FAKE.len() + slots * IMAGE.len());
+            image_string.push_str(FAKE);
+            image_string.extend(iter::repeat(IMAGE).take(slots));
+            image_string.push_str(FAKE);
+
+            if matches!(
+                preprocessor_config,
+                Some(Idefics2Processor(Idefics2Preprocessor {
+                    do_image_splitting: true,
+                    ..
+                }))
+            ) {
+                image_string = image_string.repeat(5);
+            };
+
+            image_string
+        }
+        Paligemma(config) => "<image>".repeat(config.get_number_of_features(height, width)),
+        LlavaNext(config) => "<image>".repeat(config.get_number_of_features(height, width)),
+        _ => unimplemented!("Images tokens are not supported for this model configuration"),
+    }
+}
+
+fn image_tokens_fixup(config: &Config, text: String) -> String {
+    match config {
+        Config::Idefics2(_) => {
+            const FAKE: &str = "<fake_token_around_image>";
+            text.replace(&format!("{FAKE}{FAKE}"), FAKE)
+        }
+        _ => text,
+    }
+}
+
 /// Get input length and optionally truncate it
 fn prepare_input(
-    mut inputs: String,
+    inputs: String,
     _truncate: Option<usize>,
+    add_special_tokens: bool,
     tokenizer: &Tokenizer,
-    config: &Option<Config>,
-) -> Result<(tokenizers::Encoding, String), ValidationError> {
+    config: Option<&Config>,
+    preprocessor_config: Option<&HubPreprocessorConfig>,
+) -> Result<(tokenizers::Encoding, Vec<Chunk>), ValidationError> {
+    use Config::*;
     static RE: Lazy<Regex> = Lazy::new(|| Regex::new(r"!\[\]\([^\)]*\)").unwrap());
-    let tokenizer_query = match config {
-        Some(Config::LlavaNext(config)) => {
-            let mut modified_inputs = String::with_capacity(inputs.len());
+    let (tokenizer_query, input_chunks) = match config {
+        Some(config @ (Idefics | Mllama | Idefics2(_) | Paligemma(_) | LlavaNext(_))) => {
+            let mut input_chunks = Vec::new();
             let mut tokenizer_query = String::with_capacity(inputs.len());
             let mut start = 0;
             for chunk in RE.find_iter(&inputs) {
                 let chunk_start = chunk.start();
                 let chunk_end = chunk.end();
                 if chunk_start != start {
-                    modified_inputs.push_str(&inputs[start..chunk_start]);
+                    input_chunks.push(Chunk::Text(inputs[start..chunk_start].to_string()));
                     tokenizer_query.push_str(&inputs[start..chunk_start]);
                 }
-                let (image_uri, height, width) = fetch_image(&inputs[chunk_start..chunk_end])?;
-                let slots = config.get_number_of_features(height, width);
-                tokenizer_query.push_str(&"<image>".repeat(slots));
-                modified_inputs.push_str(&image_uri);
+                let (data, mimetype, height, width) = fetch_image(&inputs[chunk_start..chunk_end])?;
+                input_chunks.push(Chunk::Image(Image { data, mimetype }));
+                tokenizer_query.push_str(&image_tokens(config, preprocessor_config, height, width));
                 start = chunk_end;
             }
-            if start != inputs.len() - 1 {
-                modified_inputs.push_str(&inputs[start..]);
+            if start != inputs.len() {
+                input_chunks.push(Chunk::Text(inputs[start..].to_string()));
                 tokenizer_query.push_str(&inputs[start..]);
             }
-            inputs = modified_inputs;
-            tokenizer_query
-        }
-        Some(Config::Paligemma(config)) => {
-            let mut modified_inputs = String::with_capacity(inputs.len());
-            let mut tokenizer_query = String::with_capacity(inputs.len());
-            let mut start = 0;
-            for chunk in RE.find_iter(&inputs) {
-                let chunk_start = chunk.start();
-                let chunk_end = chunk.end();
-                if chunk_start != start {
-                    modified_inputs.push_str(&inputs[start..chunk_start]);
-                    tokenizer_query.push_str(&inputs[start..chunk_start]);
-                }
-                let (image_uri, height, width) = fetch_image(&inputs[chunk_start..chunk_end])?;
-                let slots = config.get_number_of_features(height, width);
-                tokenizer_query.push_str(&"<image>".repeat(slots));
-                modified_inputs.push_str(&image_uri);
-                start = chunk_end;
-            }
-            if start != inputs.len() - 1 {
-                modified_inputs.push_str(&inputs[start..]);
-                tokenizer_query.push_str(&inputs[start..]);
-            }
-            inputs = modified_inputs;
-            tokenizer_query
-        }
-        Some(Config::Idefics2(config)) => {
-            let mut modified_inputs = String::with_capacity(inputs.len());
-            let mut tokenizer_query = String::with_capacity(inputs.len());
-            let mut start = 0;
-            for chunk in RE.find_iter(&inputs) {
-                let chunk_start = chunk.start();
-                let chunk_end = chunk.end();
-                if chunk_start != start {
-                    modified_inputs.push_str(&inputs[start..chunk_start]);
-                    tokenizer_query.push_str(&inputs[start..chunk_start]);
-                }
-                let (image_uri, height, width) = fetch_image(&inputs[chunk_start..chunk_end])?;
-                let slots = config.get_number_of_features(height, width);
-                tokenizer_query.push_str("<fake_token_around_image>");
-                tokenizer_query.push_str(&"<image>".repeat(slots));
-                tokenizer_query.push_str("<fake_token_around_image>");
 
-                modified_inputs.push_str(&image_uri);
-                start = chunk_end;
-            }
-            if start != inputs.len() - 1 {
-                modified_inputs.push_str(&inputs[start..]);
-                tokenizer_query.push_str(&inputs[start..]);
-            }
-            inputs = modified_inputs;
-            tokenizer_query
+            tokenizer_query = image_tokens_fixup(config, tokenizer_query);
+
+            (tokenizer_query, input_chunks)
         }
-        Some(Config::Idefics) => {
-            let mut modified_inputs = String::with_capacity(inputs.len());
-            let mut tokenizer_query = String::with_capacity(inputs.len());
-            let mut start = 0;
-            for chunk in RE.find_iter(&inputs) {
-                let chunk_start = chunk.start();
-                let chunk_end = chunk.end();
-                if chunk_start != start {
-                    modified_inputs.push_str(&inputs[start..chunk_start]);
-                    tokenizer_query.push_str(&inputs[start..chunk_start]);
-                }
-                let (image_uri, _height, _width) = fetch_image(&inputs[chunk_start..chunk_end])?;
-                let slots = 1;
-                tokenizer_query.push_str(&"<image>".repeat(slots));
-                modified_inputs.push_str(&image_uri);
-                start = chunk_end;
-            }
-            if start != inputs.len() - 1 {
-                modified_inputs.push_str(&inputs[start..]);
-                tokenizer_query.push_str(&inputs[start..]);
-            }
-            inputs = modified_inputs;
-            tokenizer_query
-        }
-        _ => inputs.clone(),
+        _ => (inputs.clone(), vec![Chunk::Text(inputs)]),
     };
 
     // Get the number of tokens in the input
     let encoding = tokenizer
-        .encode(tokenizer_query, true)
+        .encode(tokenizer_query, add_special_tokens)
         .map_err(|err| ValidationError::Tokenizer(err.to_string()))?;
 
-    Ok((encoding, inputs))
+    Ok((encoding, input_chunks))
 }
 
 type TokenizerRequest = (
-    (String, Option<usize>),
-    oneshot::Sender<Result<(tokenizers::Encoding, String), ValidationError>>,
+    (String, bool, Option<usize>),
+    oneshot::Sender<Result<(tokenizers::Encoding, Vec<Chunk>), ValidationError>>,
     Span,
 );
 
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub struct Image {
+    pub data: Vec<u8>,
+    pub mimetype: String,
+}
+
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub enum Chunk {
+    Text(String),
+    Image(Image),
+}
+
+/// Convert input chunks to a stringly-typed input for backwards
+/// compat for backends that haven't implemented chunked inputs.
+pub trait ChunksToString {
+    /// Convert chunks to string.
+    fn chunks_to_string(&self) -> String;
+}
+
+impl ChunksToString for Vec<Chunk> {
+    fn chunks_to_string(&self) -> String {
+        let mut output = String::new();
+        self.iter().for_each(|c| match &c {
+            Chunk::Text(text) => output.push_str(text),
+            Chunk::Image(Image { data, mimetype }) => {
+                let encoded = STANDARD.encode(data);
+                output.push_str(&format!("![](data:{};base64,{})", mimetype, encoded))
+            }
+        });
+        output
+    }
+}
+
 #[derive(Debug, Clone)]
-pub(crate) struct ValidGenerateRequest {
-    pub inputs: String,
+pub enum ValidGrammar {
+    Json(String),
+    Regex(String),
+}
+
+#[derive(Debug, Clone)]
+pub struct ValidParameters {
+    /// / exponential scaling output probability distribution
+    pub temperature: f32,
+    /// / restricting to the k highest probability elements
+    pub top_k: u32,
+    /// / restricting to top tokens summing to prob_cut_off <= prob_cut_off
+    pub top_p: f32,
+    /// / restricting to top tokens summing to prob_cut_off <= prob_cut_off
+    pub typical_p: f32,
+    /// / apply sampling on the logits
+    pub do_sample: bool,
+    /// / random seed for sampling
+    pub seed: u64,
+    /// / repetition penalty
+    pub repetition_penalty: f32,
+    /// / frequency penalty
+    pub frequency_penalty: f32,
+    /// / token watermarking using "A Watermark for Large Language Models"
+    pub watermark: bool,
+    /// / grammar (applied if not empty)
+    pub grammar: Option<ValidGrammar>,
+}
+
+#[derive(Debug, Clone)]
+pub struct ValidStoppingParameters {
+    /// / Maximum number of generated tokens
+    pub max_new_tokens: u32,
+    /// / Optional stopping sequences
+    pub stop_sequences: Vec<String>,
+    /// / Ignore end of sequence token
+    /// / used for benchmarking
+    pub ignore_eos_token: bool,
+}
+
+#[derive(Debug, Clone)]
+pub struct ValidGenerateRequest {
+    pub inputs: Vec<Chunk>,
+    pub input_ids: Option<Arc<Vec<u32>>>,
     pub input_length: u32,
     pub truncate: u32,
+    pub add_special_tokens: bool,
     pub decoder_input_details: bool,
-    pub parameters: NextTokenChooserParameters,
-    pub stopping_parameters: StoppingCriteriaParameters,
+    pub parameters: ValidParameters,
+    pub stopping_parameters: ValidStoppingParameters,
     pub top_n_tokens: u32,
+    pub adapter_id: Option<String>,
 }
 
 #[derive(Error, Debug)]
@@ -730,13 +811,14 @@ pub enum ValidationError {
     InvalidImageContent(String),
     #[error("Could not fetch image: {0}")]
     FailedFetchImage(#[from] reqwest::Error),
-    #[error("`watermark` = true is not allowed with FP8 quantization.")]
-    WatermarkWithQuantization,
+    #[error("{0} modality is not supported")]
+    UnsupportedModality(&'static str),
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::config::{Idefics2, PaliTextConfig, Paligemma};
     use crate::default_parameters;
     use crate::tests::get_tokenizer;
 
@@ -755,6 +837,7 @@ mod tests {
             workers,
             tokenizer,
             config,
+            None,
             max_best_of,
             max_stop_sequence,
             max_top_n_tokens,
@@ -765,10 +848,11 @@ mod tests {
 
         let max_new_tokens = 10;
         match validation
-            .validate_input("Hello".to_string(), None, Some(max_new_tokens))
+            .validate_input("Hello".to_string(), true, None, Some(max_new_tokens))
             .await
         {
-            Err(ValidationError::MaxNewTokens(1, 10)) => (),
+            // Err(ValidationError::MaxNewTokens(1, 10)) => (),
+            Ok((_s, _, 0, 10)) => (),
             r => panic!("Unexpected not max new tokens: {r:?}"),
         }
     }
@@ -788,6 +872,7 @@ mod tests {
             workers,
             tokenizer,
             config,
+            None,
             max_best_of,
             max_stop_sequence,
             max_top_n_tokens,
@@ -798,10 +883,10 @@ mod tests {
 
         let max_new_tokens = 10;
         match validation
-            .validate_input("Hello".to_string(), None, Some(max_new_tokens))
+            .validate_input("Hello".to_string(), true, None, Some(max_new_tokens))
             .await
         {
-            Err(ValidationError::MaxTotalTokens(6, 5, 10)) => (),
+            Err(ValidationError::MaxTotalTokens(6, 1, 10)) => (),
             _ => panic!("Unexpected not max new tokens"),
         }
     }
@@ -821,6 +906,7 @@ mod tests {
             workers,
             tokenizer,
             config,
+            None,
             max_best_of,
             max_stop_sequence,
             max_top_n_tokens,
@@ -831,6 +917,7 @@ mod tests {
         match validation
             .validate(GenerateRequest {
                 inputs: "Hello".to_string(),
+                add_special_tokens: true,
                 parameters: GenerateParameters {
                     best_of: Some(2),
                     do_sample: false,
@@ -859,6 +946,7 @@ mod tests {
             workers,
             tokenizer,
             config,
+            None,
             max_best_of,
             max_stop_sequence,
             max_top_n_tokens,
@@ -869,6 +957,7 @@ mod tests {
         match validation
             .validate(GenerateRequest {
                 inputs: "Hello".to_string(),
+                add_special_tokens: true,
                 parameters: GenerateParameters {
                     top_p: Some(1.0),
                     max_new_tokens: Some(5),
@@ -884,6 +973,7 @@ mod tests {
         match validation
             .validate(GenerateRequest {
                 inputs: "Hello".to_string(),
+                add_special_tokens: true,
                 parameters: GenerateParameters {
                     top_p: Some(0.99),
                     max_new_tokens: Some(5),
@@ -899,6 +989,7 @@ mod tests {
         let valid_request = validation
             .validate(GenerateRequest {
                 inputs: "Hello".to_string(),
+                add_special_tokens: true,
                 parameters: GenerateParameters {
                     top_p: None,
                     max_new_tokens: Some(5),
@@ -926,6 +1017,7 @@ mod tests {
             workers,
             tokenizer,
             config,
+            None,
             max_best_of,
             max_stop_sequences,
             max_top_n_tokens,
@@ -936,6 +1028,7 @@ mod tests {
         match validation
             .validate(GenerateRequest {
                 inputs: "Hello".to_string(),
+                add_special_tokens: true,
                 parameters: GenerateParameters {
                     top_n_tokens: Some(5),
                     max_new_tokens: Some(5),
@@ -951,6 +1044,7 @@ mod tests {
         validation
             .validate(GenerateRequest {
                 inputs: "Hello".to_string(),
+                add_special_tokens: true,
                 parameters: GenerateParameters {
                     top_n_tokens: Some(4),
                     max_new_tokens: Some(5),
@@ -963,6 +1057,7 @@ mod tests {
         validation
             .validate(GenerateRequest {
                 inputs: "Hello".to_string(),
+                add_special_tokens: true,
                 parameters: GenerateParameters {
                     top_n_tokens: Some(0),
                     max_new_tokens: Some(5),
@@ -975,6 +1070,7 @@ mod tests {
         let valid_request = validation
             .validate(GenerateRequest {
                 inputs: "Hello".to_string(),
+                add_special_tokens: true,
                 parameters: GenerateParameters {
                     top_n_tokens: None,
                     max_new_tokens: Some(5),
@@ -986,4 +1082,143 @@ mod tests {
 
         assert_eq!(valid_request.top_n_tokens, 0);
     }
+
+    static PIXEL_GIF: &str = "R0lGODdhAQABAIEAAP///wAAAAAAAAAAACwAAAAAAQABAAAIBAABBAQAOw==";
+
+    #[tokio::test]
+    async fn test_prepare_input_chunks() {
+        let pixel_data = STANDARD.decode(PIXEL_GIF).unwrap();
+
+        let tokenizer = Some(get_tokenizer().await);
+
+        let max_best_of = 2;
+        let max_stop_sequence = 3;
+        let max_top_n_tokens = 4;
+        let max_input_length = 5;
+        let max_total_tokens = 6;
+        let disable_grammar_support = true;
+        let workers = 1;
+        let config = Config::Paligemma(Paligemma {
+            text_config: PaliTextConfig {
+                num_image_tokens: 1,
+            },
+        });
+        let validation = Validation::new(
+            workers,
+            tokenizer,
+            Some(config),
+            None,
+            max_best_of,
+            max_stop_sequence,
+            max_top_n_tokens,
+            max_input_length,
+            max_total_tokens,
+            disable_grammar_support,
+        );
+
+        let chunks = match validation
+            .tokenize(
+                format!("test![](data:image/gif;base64,{})", PIXEL_GIF),
+                true,
+                None,
+            )
+            .await
+        {
+            Ok(Some((_encoding, chunks))) => chunks,
+            _ => panic!("Unexpected tokenization failure"),
+        };
+
+        assert!(
+            chunks
+                == vec![
+                    Chunk::Text("test".to_string()).into(),
+                    Chunk::Image(Image {
+                        data: pixel_data.clone(),
+                        mimetype: "image/gif".to_string()
+                    })
+                    .into()
+                ],
+            "Failed to process images",
+        );
+    }
+
+    #[tokio::test]
+    async fn test_idefics2_correct_n_fake_tokens() {
+        let pixel_data = STANDARD.decode(PIXEL_GIF).unwrap();
+
+        let tokenizer = Some(get_tokenizer().await);
+
+        let max_best_of = 2;
+        let max_stop_sequence = 3;
+        let max_top_n_tokens = 4;
+        let max_input_length = 5;
+        let max_total_tokens = 6;
+        let disable_grammar_support = true;
+        let workers = 1;
+        let config = Config::Idefics2(Idefics2 {});
+        let validation = Validation::new(
+            workers,
+            tokenizer,
+            Some(config),
+            Some(HubPreprocessorConfig::Idefics2Processor(
+                Idefics2Preprocessor {
+                    do_image_splitting: true,
+                },
+            )),
+            max_best_of,
+            max_stop_sequence,
+            max_top_n_tokens,
+            max_input_length,
+            max_total_tokens,
+            disable_grammar_support,
+        );
+
+        let (encoding, chunks) = match validation
+            .tokenize(
+                format!(
+                    "test![](data:image/gif;base64,{})![](data:image/gif;base64,{})",
+                    PIXEL_GIF, PIXEL_GIF
+                ),
+                true,
+                None,
+            )
+            .await
+        {
+            Ok(Some((encoding, chunks))) => (encoding, chunks),
+            _ => panic!("Unexpected tokenization failure"),
+        };
+
+        assert!(
+            chunks
+                == vec![
+                    Chunk::Text("test".to_string()).into(),
+                    Chunk::Image(Image {
+                        data: pixel_data.clone(),
+                        mimetype: "image/gif".to_string()
+                    })
+                    .into(),
+                    Chunk::Image(Image {
+                        data: pixel_data.clone(),
+                        mimetype: "image/gif".to_string()
+                    })
+                    .into()
+                ],
+            "Failed to process images",
+        );
+
+        // Verify the number of fake tokens:
+        //
+        // - Two images surrounded/separated by a fake token = 3.
+        // - Both are split in 5 subimages, separated by a fake token: 2 * 4
+        //
+        // Fake tokens get split up by the testing tokenizer, but we don't care.
+        assert_eq!(
+            encoding
+                .get_tokens()
+                .iter()
+                .filter(|t| *t == "fake")
+                .count(),
+            11
+        );
+    }
 }
diff --git a/router/src/vertex.rs b/router/src/vertex.rs
new file mode 100644
index 00000000..0c1467fe
--- /dev/null
+++ b/router/src/vertex.rs
@@ -0,0 +1,360 @@
+use crate::infer::Infer;
+use crate::server::{generate_internal, ComputeType};
+use crate::{
+    ChatRequest, ErrorResponse, GenerateParameters, GenerateRequest, GrammarType, Message,
+    StreamOptions, Tool, ToolChoice,
+};
+use axum::extract::Extension;
+use axum::http::{HeaderMap, StatusCode};
+use axum::response::{IntoResponse, Response};
+use axum::Json;
+use serde::{Deserialize, Serialize};
+use tracing::instrument;
+use utoipa::ToSchema;
+
+#[derive(Clone, Deserialize, ToSchema)]
+#[cfg_attr(test, derive(Debug, PartialEq))]
+pub(crate) struct GenerateVertexInstance {
+    #[schema(example = "What is Deep Learning?")]
+    pub inputs: String,
+    #[schema(nullable = true, default = "null", example = "null")]
+    pub parameters: Option<GenerateParameters>,
+}
+
+#[derive(Clone, Deserialize, ToSchema)]
+#[cfg_attr(test, derive(Debug, PartialEq))]
+pub(crate) struct VertexChat {
+    messages: Vec<Message>,
+    // Messages is ignored there.
+    #[serde(default)]
+    parameters: VertexParameters,
+}
+
+#[derive(Clone, Deserialize, ToSchema, Serialize, Default)]
+#[cfg_attr(test, derive(Debug, PartialEq))]
+pub(crate) struct VertexParameters {
+    #[schema(example = "mistralai/Mistral-7B-Instruct-v0.2")]
+    /// [UNUSED] ID of the model to use. See the model endpoint compatibility table for details on which models work with the Chat API.
+    pub model: Option<String>,
+
+    /// Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far,
+    /// decreasing the model's likelihood to repeat the same line verbatim.
+    #[serde(default)]
+    #[schema(example = "1.0")]
+    pub frequency_penalty: Option<f32>,
+
+    /// UNUSED
+    /// Modify the likelihood of specified tokens appearing in the completion. Accepts a JSON object that maps tokens
+    /// (specified by their token ID in the tokenizer) to an associated bias value from -100 to 100. Mathematically,
+    /// the bias is added to the logits generated by the model prior to sampling. The exact effect will vary per model,
+    /// but values between -1 and 1 should decrease or increase likelihood of selection; values like -100 or 100 should
+    /// result in a ban or exclusive selection of the relevant token.
+    #[serde(default)]
+    pub logit_bias: Option<Vec<f32>>,
+
+    /// Whether to return log probabilities of the output tokens or not. If true, returns the log probabilities of each
+    /// output token returned in the content of message.
+    #[serde(default)]
+    #[schema(example = "false")]
+    pub logprobs: Option<bool>,
+
+    /// An integer between 0 and 5 specifying the number of most likely tokens to return at each token position, each with
+    /// an associated log probability. logprobs must be set to true if this parameter is used.
+    #[serde(default)]
+    #[schema(example = "5")]
+    pub top_logprobs: Option<u32>,
+
+    /// The maximum number of tokens that can be generated in the chat completion.
+    #[serde(default)]
+    #[schema(example = "32")]
+    pub max_tokens: Option<u32>,
+
+    /// UNUSED
+    /// How many chat completion choices to generate for each input message. Note that you will be charged based on the
+    /// number of generated tokens across all of the choices. Keep n as 1 to minimize costs.
+    #[serde(default)]
+    #[schema(nullable = true, example = "2")]
+    pub n: Option<u32>,
+
+    /// Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far,
+    /// increasing the model's likelihood to talk about new topics
+    #[serde(default)]
+    #[schema(nullable = true, example = 0.1)]
+    pub presence_penalty: Option<f32>,
+
+    /// Up to 4 sequences where the API will stop generating further tokens.
+    #[serde(default)]
+    #[schema(nullable = true, example = "null")]
+    pub stop: Option<Vec<String>>,
+
+    #[serde(default = "bool::default")]
+    pub stream: bool,
+
+    #[schema(nullable = true, example = 42)]
+    pub seed: Option<u64>,
+
+    /// What sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the output more random, while
+    /// lower values like 0.2 will make it more focused and deterministic.
+    ///
+    /// We generally recommend altering this or `top_p` but not both.
+    #[serde(default)]
+    #[schema(nullable = true, example = 1.0)]
+    pub temperature: Option<f32>,
+
+    /// An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the
+    /// tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered.
+    #[serde(default)]
+    #[schema(nullable = true, example = 0.95)]
+    pub top_p: Option<f32>,
+
+    /// A list of tools the model may call. Currently, only functions are supported as a tool. Use this to provide a list of
+    /// functions the model may generate JSON inputs for.
+    #[serde(default)]
+    #[schema(nullable = true, example = "null")]
+    pub tools: Option<Vec<Tool>>,
+
+    /// A prompt to be appended before the tools
+    #[serde(default)]
+    #[schema(
+        nullable = true,
+        example = "Given the functions available, please respond with a JSON for a function call with its proper arguments that best answers the given prompt. Respond in the format {name: function name, parameters: dictionary of argument name and its value}.Do not use variables."
+    )]
+    pub tool_prompt: Option<String>,
+
+    /// A specific tool to use. If not provided, the model will default to use any of the tools provided in the tools parameter.
+    #[serde(default)]
+    #[schema(nullable = true, example = "null")]
+    pub tool_choice: ToolChoice,
+
+    /// Response format constraints for the generation.
+    ///
+    /// NOTE: A request can use `response_format` OR `tools` but not both.
+    #[serde(default)]
+    #[schema(nullable = true, default = "null", example = "null")]
+    pub response_format: Option<GrammarType>,
+
+    /// A guideline to be used in the chat_template
+    #[serde(default)]
+    #[schema(nullable = true, default = "null", example = "null")]
+    pub guideline: Option<String>,
+
+    /// Options for streaming response. Only set this when you set stream: true.
+    #[serde(default)]
+    #[schema(nullable = true, example = "null")]
+    pub stream_options: Option<StreamOptions>,
+}
+
+impl From<VertexChat> for ChatRequest {
+    fn from(val: VertexChat) -> Self {
+        Self {
+            messages: val.messages,
+            frequency_penalty: val.parameters.frequency_penalty,
+            guideline: val.parameters.guideline,
+            logit_bias: val.parameters.logit_bias,
+            logprobs: val.parameters.logprobs,
+            max_tokens: val.parameters.max_tokens,
+            model: val.parameters.model,
+            n: val.parameters.n,
+            presence_penalty: val.parameters.presence_penalty,
+            response_format: val.parameters.response_format,
+            seed: val.parameters.seed,
+            stop: val.parameters.stop,
+            stream_options: val.parameters.stream_options,
+            stream: val.parameters.stream,
+            temperature: val.parameters.temperature,
+            tool_choice: val.parameters.tool_choice,
+            tool_prompt: val.parameters.tool_prompt,
+            tools: val.parameters.tools,
+            top_logprobs: val.parameters.top_logprobs,
+            top_p: val.parameters.top_p,
+        }
+    }
+}
+
+#[derive(Clone, Deserialize, ToSchema)]
+#[cfg_attr(test, derive(Debug, PartialEq))]
+#[serde(untagged)]
+pub(crate) enum VertexInstance {
+    Generate(GenerateVertexInstance),
+    Chat(VertexChat),
+}
+
+#[derive(Deserialize, ToSchema)]
+#[cfg_attr(test, derive(Debug, PartialEq))]
+pub(crate) struct VertexRequest {
+    #[serde(rename = "instances")]
+    pub instances: Vec<VertexInstance>,
+}
+
+#[derive(Clone, Deserialize, ToSchema, Serialize)]
+pub(crate) struct VertexResponse {
+    pub predictions: Vec<String>,
+}
+
+/// Generate tokens from Vertex request
+#[utoipa::path(
+post,
+tag = "Text Generation Inference",
+path = "/vertex",
+request_body = VertexRequest,
+responses(
+(status = 200, description = "Generated Text", body = VertexResponse),
+(status = 424, description = "Generation Error", body = ErrorResponse,
+example = json ! ({"error": "Request failed during generation"})),
+(status = 429, description = "Model is overloaded", body = ErrorResponse,
+example = json ! ({"error": "Model is overloaded"})),
+(status = 422, description = "Input validation error", body = ErrorResponse,
+example = json ! ({"error": "Input validation error"})),
+(status = 500, description = "Incomplete generation", body = ErrorResponse,
+example = json ! ({"error": "Incomplete generation"})),
+)
+)]
+#[instrument(
+    skip_all,
+    fields(
+        total_time,
+        validation_time,
+        queue_time,
+        inference_time,
+        time_per_token,
+        seed,
+    )
+)]
+pub(crate) async fn vertex_compatibility(
+    Extension(infer): Extension<Infer>,
+    Extension(compute_type): Extension<ComputeType>,
+    Json(req): Json<VertexRequest>,
+) -> Result<Response, (StatusCode, Json<ErrorResponse>)> {
+    let span = tracing::Span::current();
+    metrics::counter!("tgi_request_count").increment(1);
+
+    // check that theres at least one instance
+    if req.instances.is_empty() {
+        return Err((
+            StatusCode::UNPROCESSABLE_ENTITY,
+            Json(ErrorResponse {
+                error: "Input validation error".to_string(),
+                error_type: "Input validation error".to_string(),
+            }),
+        ));
+    }
+
+    // Prepare futures for all instances
+    let mut futures = Vec::with_capacity(req.instances.len());
+
+    for instance in req.instances.into_iter() {
+        let generate_request = match instance {
+            VertexInstance::Generate(instance) => GenerateRequest {
+                inputs: instance.inputs.clone(),
+                add_special_tokens: true,
+                parameters: GenerateParameters {
+                    do_sample: true,
+                    max_new_tokens: instance.parameters.as_ref().and_then(|p| p.max_new_tokens),
+                    seed: instance.parameters.as_ref().and_then(|p| p.seed),
+                    details: true,
+                    decoder_input_details: true,
+                    ..Default::default()
+                },
+            },
+            VertexInstance::Chat(instance) => {
+                let chat_request: ChatRequest = instance.into();
+                let (generate_request, _using_tools): (GenerateRequest, bool) =
+                    chat_request.try_into_generate(&infer)?;
+                generate_request
+            }
+        };
+
+        let infer_clone = infer.clone();
+        let compute_type_clone = compute_type.clone();
+        let span_clone = span.clone();
+
+        futures.push(async move {
+            generate_internal(
+                Extension(infer_clone),
+                compute_type_clone,
+                Json(generate_request),
+                span_clone,
+            )
+            .await
+            .map(|(_, Json(generation))| generation.generated_text)
+            .map_err(|_| {
+                (
+                    StatusCode::INTERNAL_SERVER_ERROR,
+                    Json(ErrorResponse {
+                        error: "Incomplete generation".into(),
+                        error_type: "Incomplete generation".into(),
+                    }),
+                )
+            })
+        });
+    }
+
+    // execute all futures in parallel, collect results, returning early if any error occurs
+    let results = futures::future::join_all(futures).await;
+    let predictions: Result<Vec<_>, _> = results.into_iter().collect();
+    let predictions = predictions?;
+
+    let response = VertexResponse { predictions };
+    Ok((HeaderMap::new(), Json(response)).into_response())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::{Message, MessageContent};
+
+    #[test]
+    fn vertex_deserialization() {
+        let string = serde_json::json!({
+
+            "messages": [{"role": "user", "content": "What's Deep Learning?"}],
+            "parameters": {
+                "max_tokens": 128,
+                "top_p": 0.95,
+                "temperature": 0.7
+            }
+        });
+
+        let _request: VertexChat = serde_json::from_value(string).expect("Can deserialize");
+
+        let string = serde_json::json!({
+            "messages": [{"role": "user", "content": "What's Deep Learning?"}],
+        });
+
+        let _request: VertexChat = serde_json::from_value(string).expect("Can deserialize");
+
+        let string = serde_json::json!({
+
+        "instances": [
+            {
+                "messages": [{"role": "user", "content": "What's Deep Learning?"}],
+                "parameters": {
+                    "max_tokens": 128,
+                    "top_p": 0.95,
+                    "temperature": 0.7
+                }
+            }
+        ]
+
+        });
+        let request: VertexRequest = serde_json::from_value(string).expect("Can deserialize");
+        assert_eq!(
+            request,
+            VertexRequest {
+                instances: vec![VertexInstance::Chat(VertexChat {
+                    messages: vec![Message {
+                        role: "user".to_string(),
+                        content: MessageContent::SingleText("What's Deep Learning?".to_string()),
+                        name: None,
+                    },],
+                    parameters: VertexParameters {
+                        max_tokens: Some(128),
+                        top_p: Some(0.95),
+                        temperature: Some(0.7),
+                        ..Default::default()
+                    }
+                })]
+            }
+        );
+    }
+}
diff --git a/rust-toolchain.toml b/rust-toolchain.toml
index 507ee859..f392b161 100644
--- a/rust-toolchain.toml
+++ b/rust-toolchain.toml
@@ -1,5 +1,5 @@
 [toolchain]
-# Released on: 02 May, 2024
-# https://releases.rs/docs/1.78.0/
-channel = "1.78.0"
+# Released on: June 13, 2024
+# https://releases.rs/docs/1.79.0/
+channel = "1.80.0"
 components = ["rustfmt", "clippy"]
diff --git a/server/Makefile b/server/Makefile
index 7e38eb12..f01897e5 100644
--- a/server/Makefile
+++ b/server/Makefile
@@ -10,16 +10,16 @@ unit-tests:
 
 gen-server:
 	# Compile protos
-	pip install grpcio-tools==1.51.1 mypy-protobuf==3.4.0 'types-protobuf>=3.20.4' --no-cache-dir
+	pip install grpcio-tools==1.62.2 mypy-protobuf==3.6.0 'types-protobuf' --no-cache-dir
 	mkdir text_generation_server/pb || true
-	python -m grpc_tools.protoc -I../proto --python_out=text_generation_server/pb \
-		--grpc_python_out=text_generation_server/pb --mypy_out=text_generation_server/pb ../proto/generate.proto
+	python -m grpc_tools.protoc -I../proto/v3 --python_out=text_generation_server/pb \
+		--grpc_python_out=text_generation_server/pb --mypy_out=text_generation_server/pb ../proto/v3/generate.proto
 	find text_generation_server/pb/ -type f -name "*.py" -print0 -exec sed -i -e 's/^\(import.*pb2\)/from . \1/g' {} \;
 	touch text_generation_server/pb/__init__.py
 
 install: gen-server
 	pip install pip --upgrade
-	pip install -r requirements.txt
+	pip install --no-deps -r requirements.txt
 	pip install -e "."
 
 run-dev:
diff --git a/server/Makefile-exllamav2 b/server/Makefile-exllamav2
new file mode 100644
index 00000000..0d4cc385
--- /dev/null
+++ b/server/Makefile-exllamav2
@@ -0,0 +1,12 @@
+exllamav2_commit := v0.1.8
+
+build-exllamav2:
+	git clone https://github.com/turboderp/exllamav2.git exllamav2 && \
+	cd exllamav2 && git fetch && git checkout $(exllamav2_commit)  && \
+	git submodule update --init --recursive && \
+	pip install -r requirements.txt && \
+	CUDA_ARCH_LIST="8.0;9.0a" NVCC_GENCODE="-gencode=arch=compute_80,code=sm_80 -gencode=arch=compute_90a,code=sm_90a" TORCH_CUDA_ARCH_LIST="8.0;9.0a" python setup.py build
+
+install-exllamav2: build-exllamav2
+	cd exllamav2/ &&  \
+	CUDA_ARCH_LIST="8.0;9.0a" NVCC_GENCODE="-gencode=arch=compute_80,code=sm_80 -gencode=arch=compute_90a,code=sm_90a" TORCH_CUDA_ARCH_LIST="8.0;9.0a" python setup.py install
diff --git a/server/Makefile-fbgemm b/server/Makefile-fbgemm
new file mode 100644
index 00000000..3b8061a1
--- /dev/null
+++ b/server/Makefile-fbgemm
@@ -0,0 +1,15 @@
+fbgemm_commit := v0.8.0
+
+build-fbgemm:
+	@if [ ! -d "fbgemm" ]; then \
+		git clone https://github.com/pytorch/FBGEMM.git fbgemm; \
+	fi
+	cd fbgemm && git fetch && git checkout $(fbgemm_commit)  && \
+	git submodule update --init --recursive && \
+	cd fbgemm_gpu && \
+	pip install -r requirements.txt && \
+	CUDA_ARCH_LIST="8.0;9.0a" NVCC_GENCODE="-gencode=arch=compute_80,code=sm_80 -gencode=arch=compute_90a,code=sm_90a" TORCH_CUDA_ARCH_LIST="8.0;9.0a" python setup.py --package_variant genai build
+
+install-fbgemm: build-fbgemm
+	cd fbgemm/fbgemm_gpu &&  \
+	CUDA_ARCH_LIST="8.0;9.0a" NVCC_GENCODE="-gencode=arch=compute_80,code=sm_80 -gencode=arch=compute_90a,code=sm_90a" TORCH_CUDA_ARCH_LIST="8.0;9.0a" python setup.py --package_variant genai install
diff --git a/server/Makefile-flash-att b/server/Makefile-flash-att
index ffa304aa..29e75bc4 100644
--- a/server/Makefile-flash-att
+++ b/server/Makefile-flash-att
@@ -1,16 +1,12 @@
 flash_att_commit := 3a9bfd076f98746c73362328958dbc68d145fbec
 
-flash-attention:
-    # Clone flash attention
-	pip install -U packaging ninja  --no-cache-dir
-	git clone https://github.com/HazyResearch/flash-attention.git
-
-build-flash-attention: flash-attention
-	cd flash-attention && git fetch && git checkout $(flash_att_commit)
-	cd flash-attention && python setup.py build
-	cd flash-attention/csrc/rotary && python setup.py build
-	cd flash-attention/csrc/layer_norm && python setup.py build
+build-flash-attention:
+	if [ ! -d 'flash-attention' ]; then \
+		pip install -U packaging ninja  --no-cache-dir && \
+		git clone https://github.com/HazyResearch/flash-attention.git; \
+	fi
+	cd flash-attention && git fetch && git checkout $(flash_att_commit) && \
+	MAX_JOBS=8 python setup.py build && cd csrc/layer_norm && python setup.py build && cd ../rotary && python setup.py build
 
 install-flash-attention: build-flash-attention
-	pip uninstall flash_attn rotary_emb dropout_layer_norm -y || true
-	cd flash-attention && python setup.py install && cd csrc/layer_norm && python setup.py install && cd ../rotary && python setup.py install
+	cd flash-attention && git checkout $(flash_att_commit) && MAX_JOBS=8 python setup.py install && cd csrc/layer_norm && python setup.py install && cd ../rotary && python setup.py install
diff --git a/server/Makefile-flash-att-v2 b/server/Makefile-flash-att-v2
index 36ef576a..a9cdf782 100644
--- a/server/Makefile-flash-att-v2
+++ b/server/Makefile-flash-att-v2
@@ -1,29 +1,21 @@
-flash_att_v2_commit_cuda := 23e8fa5a263d1c7122bc46a86ef32030ee7130f9
-flash_att_v2_commit_rocm := 2554f490101742ccdc56620a938f847f61754be6
+flash_att_v2_commit_cuda := v2.6.1
+flash_att_v2_commit_rocm := 2092111b9f975b3347c652ff7fabd431130256c4
 
-
-flash-attention-v2-cuda:
-  # Clone flash attention
-	pip install -U packaging ninja  --no-cache-dir
-	git clone https://github.com/HazyResearch/flash-attention.git flash-attention-v2
-
-build-flash-attention-v2-cuda: flash-attention-v2-cuda
-	cd flash-attention-v2 && git fetch && git checkout $(flash_att_v2_commit_cuda)
-	cd flash-attention-v2 && git submodule update --init --recursive
-	cd flash-attention-v2 && python setup.py build
+build-flash-attention-v2-cuda:
+	pip install -U packaging wheel
+	pip install flash-attn==$(flash_att_v2_commit_cuda)
 
 install-flash-attention-v2-cuda: build-flash-attention-v2-cuda
-	cd flash-attention-v2 && git submodule update --init --recursive && python setup.py install
+	echo "Flash v2 installed"
 
-flash-attention-v2-rocm:
-  # Clone flash attention
-	pip install -U packaging ninja  --no-cache-dir
-	git clone https://github.com/ROCm/flash-attention.git flash-attention-v2
-
-build-flash-attention-v2-rocm: flash-attention-v2-rocm
-	cd flash-attention-v2 && git fetch && git checkout $(flash_att_v2_commit_rocm)
-	cd flash-attention-v2 && git submodule update --init --recursive
-	cd flash-attention-v2 && GPU_ARCHS="gfx90a;gfx942" PYTORCH_ROCM_ARCH="gfx90a;gfx942" python setup.py build
+build-flash-attention-v2-rocm:
+	if [ ! -d 'flash-attention-v2' ]; then \
+		pip install -U packaging ninja  --no-cache-dir && \
+		git clone https://github.com/mht-sharma/flash-attention.git flash-attention-v2 && \
+		cd flash-attention-v2 && git fetch && git checkout $(flash_att_v2_commit_rocm) && \
+		git submodule update --init --recursive && GPU_ARCHS="gfx90a;gfx942" PYTORCH_ROCM_ARCH="gfx90a;gfx942" python setup.py build; \
+	fi
 
 install-flash-attention-v2-rocm: build-flash-attention-v2-rocm
-	cd flash-attention-v2 && git submodule update --init --recursive && python setup.py install
+	cd flash-attention-v2 &&  \
+	GPU_ARCHS="gfx90a;gfx942" PYTORCH_ROCM_ARCH="gfx90a;gfx942" python setup.py install
diff --git a/server/Makefile-flashinfer b/server/Makefile-flashinfer
new file mode 100644
index 00000000..f0a27622
--- /dev/null
+++ b/server/Makefile-flashinfer
@@ -0,0 +1,2 @@
+install-flashinfer:
+	pip install flashinfer==0.1.6 -i https://flashinfer.ai/whl/cu124/torch2.4
diff --git a/server/Makefile-lorax-punica b/server/Makefile-lorax-punica
new file mode 100644
index 00000000..72f06f76
--- /dev/null
+++ b/server/Makefile-lorax-punica
@@ -0,0 +1,12 @@
+lorax_punica_commit := c71861a653412267dc27ec86013dd945ce3474bc
+
+build-lorax-punica:
+	if [ ! -d 'lorax-punica' ]; then \
+		git clone --no-checkout https://github.com/predibase/lorax.git lorax-punica; \
+	fi
+	cd lorax-punica && git sparse-checkout set server/punica_kernels && git checkout $(lorax_punica_commit)
+	cd lorax-punica && git submodule update --init --recursive
+	cd lorax-punica/server/punica_kernels && python setup.py build
+
+install-lorax-punica: build-lorax-punica
+	cd lorax-punica/server/punica_kernels && python setup.py install
diff --git a/server/Makefile-vllm b/server/Makefile-vllm
index 62fa413f..18dcc4a0 100644
--- a/server/Makefile-vllm
+++ b/server/Makefile-vllm
@@ -1,25 +1,23 @@
-vllm-cuda:
-    # Clone vllm
-	pip install -U ninja packaging --no-cache-dir
-	git clone https://github.com/Narsil/vllm.git vllm
-
-build-vllm-cuda: vllm-cuda
-	cd vllm && git fetch && git checkout b5dfc61db88a81069e45b44f7cc99bd9e62a60fa
-	cd vllm && python setup.py build
+commit_cuda := d243e9dc7e2c9c2e36a4150ec8e64809cb55c01b
+commit_rocm := 4e0929e6e4fa0a3d09d358715c288020ea9dc247
+build-vllm-cuda:
+	if [ ! -d 'vllm' ]; then \
+		pip install -U ninja packaging --no-cache-dir && \
+		git clone https://github.com/Narsil/vllm.git vllm; \
+	fi
+	cd vllm  && git fetch origin && git checkout $(commit_cuda) && python setup.py build
 
 install-vllm-cuda: build-vllm-cuda
-	pip uninstall vllm -y || true
-	cd vllm && python setup.py install
+	cd vllm  && git fetch origin && git checkout $(commit_cuda) && pip install -e .
 
-vllm-rocm:
-    # Clone vllm
-	pip install -U ninja packaging --no-cache-dir
-	git clone https://github.com/fxmarty/rocm-vllm.git vllm
-
-build-vllm-rocm: vllm-rocm
-	cd vllm && git fetch && git checkout ca6913b3c2ffacdcb7d15e914dc34adbc6c89479
-	cd vllm && PYTORCH_ROCM_ARCH="gfx90a;gfx942" python setup.py install
+build-vllm-rocm:
+	if [ ! -d 'vllm' ]; then \
+		pip install -U ninja packaging --no-cache-dir && \
+		git clone https://github.com/mht-sharma/vllm.git vllm; \
+	fi
+	cd vllm && git fetch && git checkout $(commit_rocm) &&  \
+	PYTORCH_ROCM_ARCH="gfx90a;gfx942" python setup.py build
 
 install-vllm-rocm: build-vllm-rocm
-	pip uninstall vllm -y || true
-	cd vllm && python setup.py install
+	cd vllm && git fetch && git checkout $(commit_rocm) && \
+	PYTORCH_ROCM_ARCH="gfx90a;gfx942" pip install -e .
diff --git a/server/custom_kernels/setup.py b/server/custom_kernels/setup.py
index 69f6b72a..e0b83987 100644
--- a/server/custom_kernels/setup.py
+++ b/server/custom_kernels/setup.py
@@ -1,10 +1,7 @@
 from setuptools import setup
 from torch.utils.cpp_extension import BuildExtension, CUDAExtension
-import torch
 
 extra_compile_args = ["-std=c++17"]
-if not torch.version.hip:
-    extra_compile_args.append("-arch=compute_80")
 
 setup(
     name="custom_kernels",
diff --git a/server/exllama_kernels/setup.py b/server/exllama_kernels/setup.py
index 987d181e..cc307bf0 100644
--- a/server/exllama_kernels/setup.py
+++ b/server/exllama_kernels/setup.py
@@ -1,5 +1,17 @@
 from setuptools import setup
 from torch.utils.cpp_extension import BuildExtension, CUDAExtension
+import torch
+
+extra_cuda_cflags = []
+extra_cflags = []
+if torch.version.hip:
+    extra_cflags = ["-DLEGACY_HIPBLAS_DIRECT=ON"]
+    extra_cuda_cflags = ["-DLEGACY_HIPBLAS_DIRECT=ON"]
+
+extra_compile_args = {
+    "cxx": extra_cflags,
+    "nvcc": extra_cuda_cflags,
+}
 
 setup(
     name="exllama_kernels",
@@ -13,6 +25,7 @@ setup(
                 "exllama_kernels/cuda_func/q4_matmul.cu",
                 "exllama_kernels/cuda_func/q4_matrix.cu",
             ],
+            extra_compile_args=extra_compile_args,
         )
     ],
     cmdclass={"build_ext": BuildExtension},
diff --git a/server/exllamav2_kernels/setup.py b/server/exllamav2_kernels/setup.py
index 4a16b546..56ffa973 100644
--- a/server/exllamav2_kernels/setup.py
+++ b/server/exllamav2_kernels/setup.py
@@ -3,11 +3,13 @@ from torch.utils.cpp_extension import BuildExtension, CUDAExtension
 import torch
 
 extra_cuda_cflags = ["-lineinfo", "-O3"]
-
+extra_cflags = []
 if torch.version.hip:
-    extra_cuda_cflags += ["-DHIPBLAS_USE_HIP_HALF"]
+    extra_cflags = ["-DLEGACY_HIPBLAS_DIRECT=ON"]
+    extra_cuda_cflags += ["-DHIPBLAS_USE_HIP_HALF", "-DLEGACY_HIPBLAS_DIRECT=ON"]
 
 extra_compile_args = {
+    "cxx": extra_cflags,
     "nvcc": extra_cuda_cflags,
 }
 
diff --git a/server/poetry.lock b/server/poetry.lock
index 6378b7d5..1987c3b3 100644
--- a/server/poetry.lock
+++ b/server/poetry.lock
@@ -2,18 +2,18 @@
 
 [[package]]
 name = "accelerate"
-version = "0.33.0"
+version = "0.29.3"
 description = "Accelerate"
-optional = false
+optional = true
 python-versions = ">=3.8.0"
 files = [
-    {file = "accelerate-0.33.0-py3-none-any.whl", hash = "sha256:0a7f33d60ba09afabd028d4f0856dd19c5a734b7a596d637d9dd6e3d0eadbaf3"},
-    {file = "accelerate-0.33.0.tar.gz", hash = "sha256:11ba481ed6ea09191775df55ce464aeeba67a024bd0261a44b77b30fb439e26a"},
+    {file = "accelerate-0.29.3-py3-none-any.whl", hash = "sha256:99d633d4b6126817c5e554487406748be95c8d1d1e659dd2fd60657e35f532dd"},
+    {file = "accelerate-0.29.3.tar.gz", hash = "sha256:1a5a845b06b24b41736b219b2b20fd021ca5dff4070a252445fd6de736e347ac"},
 ]
 
 [package.dependencies]
-huggingface-hub = ">=0.21.0"
-numpy = ">=1.17,<2.0.0"
+huggingface-hub = "*"
+numpy = ">=1.17"
 packaging = ">=20.0"
 psutil = "*"
 pyyaml = "*"
@@ -21,21 +21,20 @@ safetensors = ">=0.3.1"
 torch = ">=1.10.0"
 
 [package.extras]
-deepspeed = ["deepspeed (<=0.14.0)"]
-dev = ["bitsandbytes", "black (>=23.1,<24.0)", "datasets", "diffusers", "evaluate", "hf-doc-builder (>=0.3.0)", "parameterized", "pytest (>=7.2.0,<=8.0.0)", "pytest-subtests", "pytest-xdist", "rich", "ruff (>=0.2.1,<0.3.0)", "scikit-learn", "scipy", "timm", "torchpippy (>=0.2.0)", "tqdm", "transformers"]
+dev = ["bitsandbytes", "black (>=23.1,<24.0)", "datasets", "deepspeed", "evaluate", "hf-doc-builder (>=0.3.0)", "parameterized", "pytest (>=7.2.0,<=8.0.0)", "pytest-subtests", "pytest-xdist", "rich", "ruff (>=0.2.1,<0.3.0)", "scikit-learn", "scipy", "timm", "torchpippy (>=0.2.0)", "tqdm", "transformers"]
 quality = ["black (>=23.1,<24.0)", "hf-doc-builder (>=0.3.0)", "ruff (>=0.2.1,<0.3.0)"]
 rich = ["rich"]
 sagemaker = ["sagemaker"]
-test-dev = ["bitsandbytes", "datasets", "diffusers", "evaluate", "scikit-learn", "scipy", "timm", "torchpippy (>=0.2.0)", "tqdm", "transformers"]
+test-dev = ["bitsandbytes", "datasets", "deepspeed", "evaluate", "scikit-learn", "scipy", "timm", "torchpippy (>=0.2.0)", "tqdm", "transformers"]
 test-prod = ["parameterized", "pytest (>=7.2.0,<=8.0.0)", "pytest-subtests", "pytest-xdist"]
 test-trackers = ["comet-ml", "dvclive", "tensorboard", "wandb"]
-testing = ["bitsandbytes", "datasets", "diffusers", "evaluate", "parameterized", "pytest (>=7.2.0,<=8.0.0)", "pytest-subtests", "pytest-xdist", "scikit-learn", "scipy", "timm", "torchpippy (>=0.2.0)", "tqdm", "transformers"]
+testing = ["bitsandbytes", "datasets", "deepspeed", "evaluate", "parameterized", "pytest (>=7.2.0,<=8.0.0)", "pytest-subtests", "pytest-xdist", "scikit-learn", "scipy", "timm", "torchpippy (>=0.2.0)", "tqdm", "transformers"]
 
 [[package]]
 name = "aiohappyeyeballs"
 version = "2.4.3"
 description = "Happy Eyeballs for asyncio"
-optional = false
+optional = true
 python-versions = ">=3.8"
 files = [
     {file = "aiohappyeyeballs-2.4.3-py3-none-any.whl", hash = "sha256:8a7a83727b2756f394ab2895ea0765a0a8c475e3c71e98d43d76f22b4b435572"},
@@ -46,7 +45,7 @@ files = [
 name = "aiohttp"
 version = "3.10.10"
 description = "Async http client/server framework (asyncio)"
-optional = false
+optional = true
 python-versions = ">=3.8"
 files = [
     {file = "aiohttp-3.10.10-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:be7443669ae9c016b71f402e43208e13ddf00912f47f623ee5994e12fc7d4b3f"},
@@ -158,7 +157,7 @@ speedups = ["Brotli", "aiodns (>=3.2.0)", "brotlicffi"]
 name = "aiosignal"
 version = "1.3.1"
 description = "aiosignal: a list of registered asynchronous callbacks"
-optional = false
+optional = true
 python-versions = ">=3.7"
 files = [
     {file = "aiosignal-1.3.1-py3-none-any.whl", hash = "sha256:f8376fb07dd1e86a584e4fcdec80b36b7f81aac666ebc724e2c090300dd83b17"},
@@ -183,7 +182,7 @@ files = [
 name = "async-timeout"
 version = "4.0.3"
 description = "Timeout context manager for asyncio programs"
-optional = false
+optional = true
 python-versions = ">=3.7"
 files = [
     {file = "async-timeout-4.0.3.tar.gz", hash = "sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f"},
@@ -194,7 +193,7 @@ files = [
 name = "attrs"
 version = "24.2.0"
 description = "Classes Without Boilerplate"
-optional = false
+optional = true
 python-versions = ">=3.7"
 files = [
     {file = "attrs-24.2.0-py3-none-any.whl", hash = "sha256:81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2"},
@@ -210,16 +209,24 @@ tests = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4
 tests-mypy = ["mypy (>=1.11.1)", "pytest-mypy-plugins"]
 
 [[package]]
-name = "backoff"
-version = "2.2.1"
-description = "Function decoration for backoff and retry"
-optional = false
-python-versions = ">=3.7,<4.0"
+name = "bitsandbytes"
+version = "0.43.3"
+description = "k-bit optimizers and matrix multiplication routines."
+optional = true
+python-versions = "*"
 files = [
-    {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"},
-    {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"},
+    {file = "bitsandbytes-0.43.3-py3-none-manylinux_2_24_x86_64.whl", hash = "sha256:cc99507c352be0715098b2c7577b690dd158972dc4ea10c7495bac104c7c79f0"},
+    {file = "bitsandbytes-0.43.3-py3-none-win_amd64.whl", hash = "sha256:257f6552f2144748a84e6c44e1f7a98f3da888f675ed74e18fd7f7eb13c6cafa"},
 ]
 
+[package.dependencies]
+numpy = "*"
+torch = "*"
+
+[package.extras]
+benchmark = ["matplotlib", "pandas"]
+test = ["scipy"]
+
 [[package]]
 name = "certifi"
 version = "2024.8.30"
@@ -381,28 +388,11 @@ files = [
     {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"},
 ]
 
-[[package]]
-name = "coloredlogs"
-version = "15.0.1"
-description = "Colored terminal output for Python's logging module"
-optional = false
-python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
-files = [
-    {file = "coloredlogs-15.0.1-py2.py3-none-any.whl", hash = "sha256:612ee75c546f53e92e70049c9dbfcc18c935a2b9a53b66085ce9ef6a6e5c0934"},
-    {file = "coloredlogs-15.0.1.tar.gz", hash = "sha256:7c991aa71a4577af2f82600d8f8f3a89f936baeaf9b50a9c197da014e5bf16b0"},
-]
-
-[package.dependencies]
-humanfriendly = ">=9.1"
-
-[package.extras]
-cron = ["capturer (>=2.4)"]
-
 [[package]]
 name = "datasets"
 version = "2.14.4"
 description = "HuggingFace community-driven open-source library of datasets"
-optional = false
+optional = true
 python-versions = ">=3.8.0"
 files = [
     {file = "datasets-2.14.4-py3-none-any.whl", hash = "sha256:29336bd316a7d827ccd4da2236596279b20ca2ac78f64c04c9483da7cbc2459b"},
@@ -457,36 +447,6 @@ wrapt = ">=1.10,<2"
 [package.extras]
 dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"]
 
-[[package]]
-name = "diffusers"
-version = "0.29.2"
-description = "State-of-the-art diffusion in PyTorch and JAX."
-optional = false
-python-versions = ">=3.8.0"
-files = [
-    {file = "diffusers-0.29.2-py3-none-any.whl", hash = "sha256:d5e9bb13c8097b4eed10df23d1294d2e5a418f53e3f89c7ef228b5b982970428"},
-    {file = "diffusers-0.29.2.tar.gz", hash = "sha256:b85f277668e22089cf68b40dd9b76940db7d24ba9cdac107533ed10ab8e4e9db"},
-]
-
-[package.dependencies]
-filelock = "*"
-huggingface-hub = ">=0.23.2"
-importlib-metadata = "*"
-numpy = "*"
-Pillow = "*"
-regex = "!=2019.12.17"
-requests = "*"
-safetensors = ">=0.3.1"
-
-[package.extras]
-dev = ["GitPython (<3.1.19)", "Jinja2", "accelerate (>=0.29.3)", "compel (==0.1.8)", "datasets", "flax (>=0.4.1)", "hf-doc-builder (>=0.3.0)", "invisible-watermark (>=0.2.0)", "isort (>=5.5.4)", "jax (>=0.4.1)", "jaxlib (>=0.4.1)", "k-diffusion (>=0.0.12)", "librosa", "parameterized", "peft (>=0.6.0)", "protobuf (>=3.20.3,<4)", "pytest", "pytest-timeout", "pytest-xdist", "requests-mock (==1.10.0)", "ruff (==0.1.5)", "safetensors (>=0.3.1)", "scipy", "sentencepiece (>=0.1.91,!=0.1.92)", "tensorboard", "torch (>=1.4)", "torchvision", "transformers (>=4.25.1)", "urllib3 (<=2.0.0)"]
-docs = ["hf-doc-builder (>=0.3.0)"]
-flax = ["flax (>=0.4.1)", "jax (>=0.4.1)", "jaxlib (>=0.4.1)"]
-quality = ["hf-doc-builder (>=0.3.0)", "isort (>=5.5.4)", "ruff (==0.1.5)", "urllib3 (<=2.0.0)"]
-test = ["GitPython (<3.1.19)", "Jinja2", "compel (==0.1.8)", "datasets", "invisible-watermark (>=0.2.0)", "k-diffusion (>=0.0.12)", "librosa", "parameterized", "pytest", "pytest-timeout", "pytest-xdist", "requests-mock (==1.10.0)", "safetensors (>=0.3.1)", "scipy", "sentencepiece (>=0.1.91,!=0.1.92)", "torchvision", "transformers (>=4.25.1)"]
-torch = ["accelerate (>=0.29.3)", "torch (>=1.4)"]
-training = ["Jinja2", "accelerate (>=0.29.3)", "datasets", "peft (>=0.6.0)", "protobuf (>=3.20.3,<4)", "tensorboard"]
-
 [[package]]
 name = "dill"
 version = "0.3.7"
@@ -512,6 +472,17 @@ files = [
     {file = "diskcache-5.6.3.tar.gz", hash = "sha256:2c3a3fa2743d8535d832ec61c2054a1641f41775aa7c556758a109941e33e4fc"},
 ]
 
+[[package]]
+name = "einops"
+version = "0.6.1"
+description = "A new flavour of deep learning operations"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "einops-0.6.1-py3-none-any.whl", hash = "sha256:99149e46cc808956b174932fe563d920db4d6e5dadb8c6ecdaa7483b7ef7cfc3"},
+    {file = "einops-0.6.1.tar.gz", hash = "sha256:f95f8d00f4ded90dbc4b19b6f98b177332614b0357dde66997f3ae5d474dc8c8"},
+]
+
 [[package]]
 name = "exceptiongroup"
 version = "1.2.2"
@@ -546,7 +517,7 @@ typing = ["typing-extensions (>=4.12.2)"]
 name = "frozenlist"
 version = "1.4.1"
 description = "A list-like structure which implements collections.abc.MutableSequence"
-optional = false
+optional = true
 python-versions = ">=3.8"
 files = [
     {file = "frozenlist-1.4.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:f9aa1878d1083b276b0196f2dfbe00c9b7e752475ed3b682025ff20c1c1f51ac"},
@@ -773,93 +744,95 @@ protobuf = ["grpcio-tools (>=1.67.0)"]
 
 [[package]]
 name = "grpcio-reflection"
-version = "1.48.2"
+version = "1.62.3"
 description = "Standard Protobuf Reflection Service for gRPC"
 optional = false
 python-versions = ">=3.6"
 files = [
-    {file = "grpcio-reflection-1.48.2.tar.gz", hash = "sha256:b687acc86c736ba8273523e1cdd5f31155dccabf7f9b2acfb62bf4e9c79d3b5a"},
-    {file = "grpcio_reflection-1.48.2-py3-none-any.whl", hash = "sha256:280bf4569149126050b587ff9177051a409ee98882028dcf0c9caa3c2d31f6fe"},
+    {file = "grpcio-reflection-1.62.3.tar.gz", hash = "sha256:cb84682933c400bddf94dd94f928d1c6570f500b6dd255973d4bfb495b82585f"},
+    {file = "grpcio_reflection-1.62.3-py3-none-any.whl", hash = "sha256:a48ef37df81a3bada78261fc92ef382f061112f989d1312398b945cc69838b9c"},
 ]
 
 [package.dependencies]
-grpcio = ">=1.48.2"
-protobuf = ">=3.12.0"
+grpcio = ">=1.62.3"
+protobuf = ">=4.21.6"
 
 [[package]]
 name = "grpcio-status"
-version = "1.48.2"
+version = "1.62.3"
 description = "Status proto mapping for gRPC"
 optional = false
 python-versions = ">=3.6"
 files = [
-    {file = "grpcio-status-1.48.2.tar.gz", hash = "sha256:53695f45da07437b7c344ee4ef60d370fd2850179f5a28bb26d8e2aa1102ec11"},
-    {file = "grpcio_status-1.48.2-py3-none-any.whl", hash = "sha256:2c33bbdbe20188b2953f46f31af669263b6ee2a9b2d38fa0d36ee091532e21bf"},
+    {file = "grpcio-status-1.62.3.tar.gz", hash = "sha256:289bdd7b2459794a12cf95dc0cb727bd4a1742c37bd823f760236c937e53a485"},
+    {file = "grpcio_status-1.62.3-py3-none-any.whl", hash = "sha256:f9049b762ba8de6b1086789d8315846e094edac2c50beaf462338b301a8fd4b8"},
 ]
 
 [package.dependencies]
 googleapis-common-protos = ">=1.5.5"
-grpcio = ">=1.48.2"
-protobuf = ">=3.12.0"
+grpcio = ">=1.62.3"
+protobuf = ">=4.21.6"
 
 [[package]]
 name = "grpcio-tools"
-version = "1.48.2"
+version = "1.62.3"
 description = "Protobuf code generator for gRPC"
 optional = false
-python-versions = ">=3.6"
+python-versions = ">=3.7"
 files = [
-    {file = "grpcio-tools-1.48.2.tar.gz", hash = "sha256:8902a035708555cddbd61b5467cea127484362decc52de03f061a1a520fe90cd"},
-    {file = "grpcio_tools-1.48.2-cp310-cp310-linux_armv7l.whl", hash = "sha256:92acc3e10ba2b0dcb90a88ae9fe1cc0ffba6868545207e4ff20ca95284f8e3c9"},
-    {file = "grpcio_tools-1.48.2-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:e5bb396d63495667d4df42e506eed9d74fc9a51c99c173c04395fe7604c848f1"},
-    {file = "grpcio_tools-1.48.2-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:84a84d601a238572d049d3108e04fe4c206536e81076d56e623bd525a1b38def"},
-    {file = "grpcio_tools-1.48.2-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:70564521e86a0de35ea9ac6daecff10cb46860aec469af65869974807ce8e98b"},
-    {file = "grpcio_tools-1.48.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bdbbe63f6190187de5946891941629912ac8196701ed2253fa91624a397822ec"},
-    {file = "grpcio_tools-1.48.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:ae56f133b05b7e5d780ef7e032dd762adad7f3dc8f64adb43ff5bfabd659f435"},
-    {file = "grpcio_tools-1.48.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f0feb4f2b777fa6377e977faa89c26359d4f31953de15e035505b92f41aa6906"},
-    {file = "grpcio_tools-1.48.2-cp310-cp310-win32.whl", hash = "sha256:80f450272316ca0924545f488c8492649ca3aeb7044d4bf59c426dcdee527f7c"},
-    {file = "grpcio_tools-1.48.2-cp310-cp310-win_amd64.whl", hash = "sha256:21ff50e321736eba22210bf9b94e05391a9ac345f26e7df16333dc75d63e74fb"},
-    {file = "grpcio_tools-1.48.2-cp36-cp36m-linux_armv7l.whl", hash = "sha256:d598ccde6338b2cfbb3124f34c95f03394209013f9b1ed4a5360a736853b1c27"},
-    {file = "grpcio_tools-1.48.2-cp36-cp36m-macosx_10_10_x86_64.whl", hash = "sha256:a43d26714933f23de93ea0bf9c86c66a6ede709b8ca32e357f9e2181703e64ae"},
-    {file = "grpcio_tools-1.48.2-cp36-cp36m-manylinux_2_17_aarch64.whl", hash = "sha256:55fdebc73fb580717656b1bafa4f8eca448726a7aa22726a6c0a7895d2f0f088"},
-    {file = "grpcio_tools-1.48.2-cp36-cp36m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8588819b22d0de3aa1951e1991cc3e4b9aa105eecf6e3e24eb0a2fc8ab958b3e"},
-    {file = "grpcio_tools-1.48.2-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9771d4d317dca029dfaca7ec9282d8afe731c18bc536ece37fd39b8a974cc331"},
-    {file = "grpcio_tools-1.48.2-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:d886a9e052a038642b3af5d18e6f2085d1656d9788e202dc23258cf3a751e7ca"},
-    {file = "grpcio_tools-1.48.2-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:d77e8b1613876e0d8fd17709509d4ceba13492816426bd156f7e88a4c47e7158"},
-    {file = "grpcio_tools-1.48.2-cp36-cp36m-win32.whl", hash = "sha256:dcaaecdd5e847de5c1d533ea91522bf56c9e6b2dc98cdc0d45f0a1c26e846ea2"},
-    {file = "grpcio_tools-1.48.2-cp36-cp36m-win_amd64.whl", hash = "sha256:0119aabd9ceedfdf41b56b9fdc8284dd85a7f589d087f2694d743f346a368556"},
-    {file = "grpcio_tools-1.48.2-cp37-cp37m-linux_armv7l.whl", hash = "sha256:189be2a9b672300ca6845d94016bdacc052fdbe9d1ae9e85344425efae2ff8ef"},
-    {file = "grpcio_tools-1.48.2-cp37-cp37m-macosx_10_10_x86_64.whl", hash = "sha256:9443f5c30bac449237c3cf99da125f8d6e6c01e17972bc683ee73b75dea95573"},
-    {file = "grpcio_tools-1.48.2-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:e0403e095b343431195db1305248b50019ad55d3dd310254431af87e14ef83a2"},
-    {file = "grpcio_tools-1.48.2-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5410d6b601d1404835e34466bd8aee37213489b36ee1aad2276366e265ff29d4"},
-    {file = "grpcio_tools-1.48.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51be91b7c7056ff9ee48b1eccd4a2840b0126230803a5e09dfc082a5b16a91c1"},
-    {file = "grpcio_tools-1.48.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:516eedd5eb7af6326050bc2cfceb3a977b9cc1144f283c43cc4956905285c912"},
-    {file = "grpcio_tools-1.48.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d18599ab572b2f15a8f3db49503272d1bb4fcabb4b4d1214ef03aca1816b20a0"},
-    {file = "grpcio_tools-1.48.2-cp37-cp37m-win32.whl", hash = "sha256:d18ef2adc05a8ef9e58ac46357f6d4ce7e43e077c7eda0a4425773461f9d0e6e"},
-    {file = "grpcio_tools-1.48.2-cp37-cp37m-win_amd64.whl", hash = "sha256:6d9753944e5a6b6b78b76ce9d2ae0fe3f748008c1849deb7fadcb64489d6553b"},
-    {file = "grpcio_tools-1.48.2-cp38-cp38-linux_armv7l.whl", hash = "sha256:3c8749dca04a8d302862ceeb1dfbdd071ee13b281395975f24405a347e5baa57"},
-    {file = "grpcio_tools-1.48.2-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:7307dd2408b82ea545ae63502ec03036b025f449568556ea9a056e06129a7a4e"},
-    {file = "grpcio_tools-1.48.2-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:072234859f6069dc43a6be8ad6b7d682f4ba1dc2e2db2ebf5c75f62eee0f6dfb"},
-    {file = "grpcio_tools-1.48.2-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6cc298fbfe584de8876a85355efbcf796dfbcfac5948c9560f5df82e79336e2a"},
-    {file = "grpcio_tools-1.48.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f75973a42c710999acd419968bc79f00327e03e855bbe82c6529e003e49af660"},
-    {file = "grpcio_tools-1.48.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:f766050e491d0b3203b6b85638015f543816a2eb7d089fc04e86e00f6de0e31d"},
-    {file = "grpcio_tools-1.48.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8e0d74403484eb77e8df2566a64b8b0b484b5c87903678c381634dd72f252d5e"},
-    {file = "grpcio_tools-1.48.2-cp38-cp38-win32.whl", hash = "sha256:cb75bac0cd43858cb759ef103fe68f8c540cb58b63dda127e710228fec3007b8"},
-    {file = "grpcio_tools-1.48.2-cp38-cp38-win_amd64.whl", hash = "sha256:cabc8b0905cedbc3b2b7b2856334fa35cce3d4bc79ae241cacd8cca8940a5c85"},
-    {file = "grpcio_tools-1.48.2-cp39-cp39-linux_armv7l.whl", hash = "sha256:e712a6d00606ad19abdeae852a7e521d6f6d0dcea843708fecf3a38be16a851e"},
-    {file = "grpcio_tools-1.48.2-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:e7e7668f89fd598c5469bb58e16bfd12b511d9947ccc75aec94da31f62bc3758"},
-    {file = "grpcio_tools-1.48.2-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:a415fbec67d4ff7efe88794cbe00cf548d0f0a5484cceffe0a0c89d47694c491"},
-    {file = "grpcio_tools-1.48.2-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d96e96ae7361aa51c9cd9c73b677b51f691f98df6086860fcc3c45852d96b0b0"},
-    {file = "grpcio_tools-1.48.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e20d7885a40e68a2bda92908acbabcdf3c14dd386c3845de73ba139e9df1f132"},
-    {file = "grpcio_tools-1.48.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:8a5614251c46da07549e24f417cf989710250385e9d80deeafc53a0ee7df6325"},
-    {file = "grpcio_tools-1.48.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ace0035766fe01a1b096aa050be9f0a9f98402317e7aeff8bfe55349be32a407"},
-    {file = "grpcio_tools-1.48.2-cp39-cp39-win32.whl", hash = "sha256:4fa4300b1be59b046492ed3c5fdb59760bc6433f44c08f50de900f9552ec7461"},
-    {file = "grpcio_tools-1.48.2-cp39-cp39-win_amd64.whl", hash = "sha256:0fb6c1c1e56eb26b224adc028a4204b6ad0f8b292efa28067dff273bbc8b27c4"},
+    {file = "grpcio-tools-1.62.3.tar.gz", hash = "sha256:7c7136015c3d62c3eef493efabaf9e3380e3e66d24ee8e94c01cb71377f57833"},
+    {file = "grpcio_tools-1.62.3-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:2f968b049c2849540751ec2100ab05e8086c24bead769ca734fdab58698408c1"},
+    {file = "grpcio_tools-1.62.3-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:0a8c0c4724ae9c2181b7dbc9b186df46e4f62cb18dc184e46d06c0ebeccf569e"},
+    {file = "grpcio_tools-1.62.3-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5782883a27d3fae8c425b29a9d3dcf5f47d992848a1b76970da3b5a28d424b26"},
+    {file = "grpcio_tools-1.62.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f3d812daffd0c2d2794756bd45a353f89e55dc8f91eb2fc840c51b9f6be62667"},
+    {file = "grpcio_tools-1.62.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:b47d0dda1bdb0a0ba7a9a6de88e5a1ed61f07fad613964879954961e36d49193"},
+    {file = "grpcio_tools-1.62.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:ca246dffeca0498be9b4e1ee169b62e64694b0f92e6d0be2573e65522f39eea9"},
+    {file = "grpcio_tools-1.62.3-cp310-cp310-win32.whl", hash = "sha256:6a56d344b0bab30bf342a67e33d386b0b3c4e65868ffe93c341c51e1a8853ca5"},
+    {file = "grpcio_tools-1.62.3-cp310-cp310-win_amd64.whl", hash = "sha256:710fecf6a171dcbfa263a0a3e7070e0df65ba73158d4c539cec50978f11dad5d"},
+    {file = "grpcio_tools-1.62.3-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:703f46e0012af83a36082b5f30341113474ed0d91e36640da713355cd0ea5d23"},
+    {file = "grpcio_tools-1.62.3-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:7cc83023acd8bc72cf74c2edbe85b52098501d5b74d8377bfa06f3e929803492"},
+    {file = "grpcio_tools-1.62.3-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7ff7d58a45b75df67d25f8f144936a3e44aabd91afec833ee06826bd02b7fbe7"},
+    {file = "grpcio_tools-1.62.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f2483ea232bd72d98a6dc6d7aefd97e5bc80b15cd909b9e356d6f3e326b6e43"},
+    {file = "grpcio_tools-1.62.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:962c84b4da0f3b14b3cdb10bc3837ebc5f136b67d919aea8d7bb3fd3df39528a"},
+    {file = "grpcio_tools-1.62.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:8ad0473af5544f89fc5a1ece8676dd03bdf160fb3230f967e05d0f4bf89620e3"},
+    {file = "grpcio_tools-1.62.3-cp311-cp311-win32.whl", hash = "sha256:db3bc9fa39afc5e4e2767da4459df82b095ef0cab2f257707be06c44a1c2c3e5"},
+    {file = "grpcio_tools-1.62.3-cp311-cp311-win_amd64.whl", hash = "sha256:e0898d412a434e768a0c7e365acabe13ff1558b767e400936e26b5b6ed1ee51f"},
+    {file = "grpcio_tools-1.62.3-cp312-cp312-macosx_10_10_universal2.whl", hash = "sha256:d102b9b21c4e1e40af9a2ab3c6d41afba6bd29c0aa50ca013bf85c99cdc44ac5"},
+    {file = "grpcio_tools-1.62.3-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:0a52cc9444df978438b8d2332c0ca99000521895229934a59f94f37ed896b133"},
+    {file = "grpcio_tools-1.62.3-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:141d028bf5762d4a97f981c501da873589df3f7e02f4c1260e1921e565b376fa"},
+    {file = "grpcio_tools-1.62.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47a5c093ab256dec5714a7a345f8cc89315cb57c298b276fa244f37a0ba507f0"},
+    {file = "grpcio_tools-1.62.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:f6831fdec2b853c9daa3358535c55eed3694325889aa714070528cf8f92d7d6d"},
+    {file = "grpcio_tools-1.62.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:e02d7c1a02e3814c94ba0cfe43d93e872c758bd8fd5c2797f894d0c49b4a1dfc"},
+    {file = "grpcio_tools-1.62.3-cp312-cp312-win32.whl", hash = "sha256:b881fd9505a84457e9f7e99362eeedd86497b659030cf57c6f0070df6d9c2b9b"},
+    {file = "grpcio_tools-1.62.3-cp312-cp312-win_amd64.whl", hash = "sha256:11c625eebefd1fd40a228fc8bae385e448c7e32a6ae134e43cf13bbc23f902b7"},
+    {file = "grpcio_tools-1.62.3-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:ec6fbded0c61afe6f84e3c2a43e6d656791d95747d6d28b73eff1af64108c434"},
+    {file = "grpcio_tools-1.62.3-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:bfda6ee8990997a9df95c5606f3096dae65f09af7ca03a1e9ca28f088caca5cf"},
+    {file = "grpcio_tools-1.62.3-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b77f9f9cee87cd798f0fe26b7024344d1b03a7cd2d2cba7035f8433b13986325"},
+    {file = "grpcio_tools-1.62.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2e02d3b96f2d0e4bab9ceaa30f37d4f75571e40c6272e95364bff3125a64d184"},
+    {file = "grpcio_tools-1.62.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:1da38070738da53556a4b35ab67c1b9884a5dd48fa2f243db35dc14079ea3d0c"},
+    {file = "grpcio_tools-1.62.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ace43b26d88a58dcff16c20d23ff72b04d0a415f64d2820f4ff06b1166f50557"},
+    {file = "grpcio_tools-1.62.3-cp37-cp37m-win_amd64.whl", hash = "sha256:350a80485e302daaa95d335a931f97b693e170e02d43767ab06552c708808950"},
+    {file = "grpcio_tools-1.62.3-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:c3a1ac9d394f8e229eb28eec2e04b9a6f5433fa19c9d32f1cb6066e3c5114a1d"},
+    {file = "grpcio_tools-1.62.3-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:11f363570dea661dde99e04a51bd108a5807b5df32a6f8bdf4860e34e94a4dbf"},
+    {file = "grpcio_tools-1.62.3-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dc9ad9950119d8ae27634e68b7663cc8d340ae535a0f80d85a55e56a6973ab1f"},
+    {file = "grpcio_tools-1.62.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8c5d22b252dcef11dd1e0fbbe5bbfb9b4ae048e8880d33338215e8ccbdb03edc"},
+    {file = "grpcio_tools-1.62.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:27cd9ef5c5d68d5ed104b6dcb96fe9c66b82050e546c9e255716903c3d8f0373"},
+    {file = "grpcio_tools-1.62.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:f4b1615adf67bd8bb71f3464146a6f9949972d06d21a4f5e87e73f6464d97f57"},
+    {file = "grpcio_tools-1.62.3-cp38-cp38-win32.whl", hash = "sha256:e18e15287c31baf574fcdf8251fb7f997d64e96c6ecf467906e576da0a079af6"},
+    {file = "grpcio_tools-1.62.3-cp38-cp38-win_amd64.whl", hash = "sha256:6c3064610826f50bd69410c63101954676edc703e03f9e8f978a135f1aaf97c1"},
+    {file = "grpcio_tools-1.62.3-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:8e62cc7164b0b7c5128e637e394eb2ef3db0e61fc798e80c301de3b2379203ed"},
+    {file = "grpcio_tools-1.62.3-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:c8ad5cce554e2fcaf8842dee5d9462583b601a3a78f8b76a153c38c963f58c10"},
+    {file = "grpcio_tools-1.62.3-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ec279dcf3518201fc592c65002754f58a6b542798cd7f3ecd4af086422f33f29"},
+    {file = "grpcio_tools-1.62.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1c989246c2aebc13253f08be32538a4039a64e12d9c18f6d662d7aee641dc8b5"},
+    {file = "grpcio_tools-1.62.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:ca4f5eeadbb57cf03317d6a2857823239a63a59cc935f5bd6cf6e8b7af7a7ecc"},
+    {file = "grpcio_tools-1.62.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:0cb3a3436ac119cbd37a7d3331d9bdf85dad21a6ac233a3411dff716dcbf401e"},
+    {file = "grpcio_tools-1.62.3-cp39-cp39-win32.whl", hash = "sha256:3eae6ea76d62fcac091e1f15c2dcedf1dc3f114f8df1a972a8a0745e89f4cf61"},
+    {file = "grpcio_tools-1.62.3-cp39-cp39-win_amd64.whl", hash = "sha256:eec73a005443061f4759b71a056f745e3b000dc0dc125c9f20560232dfbcbd14"},
 ]
 
 [package.dependencies]
-grpcio = ">=1.48.2"
-protobuf = ">=3.12.0,<4.0dev"
+grpcio = ">=1.62.3"
+protobuf = ">=4.21.6,<5.0dev"
 setuptools = "*"
 
 [[package]]
@@ -960,20 +933,6 @@ testing = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "fastapi", "gr
 torch = ["safetensors[torch]", "torch"]
 typing = ["types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)"]
 
-[[package]]
-name = "humanfriendly"
-version = "10.0"
-description = "Human friendly output for text interfaces using Python"
-optional = false
-python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
-files = [
-    {file = "humanfriendly-10.0-py2.py3-none-any.whl", hash = "sha256:1697e1a8a8f550fd43c2865cd84542fc175a61dcb779b6fee18cf6b6ccba1477"},
-    {file = "humanfriendly-10.0.tar.gz", hash = "sha256:6b0b831ce8f15f7300721aa49829fc4e83921a9a301cc7f606be6686a2288ddc"},
-]
-
-[package.dependencies]
-pyreadline3 = {version = "*", markers = "sys_platform == \"win32\" and python_version >= \"3.8\""}
-
 [[package]]
 name = "idna"
 version = "3.10"
@@ -1037,7 +996,7 @@ files = [
 name = "jinja2"
 version = "3.1.4"
 description = "A very fast and expressive template engine."
-optional = false
+optional = true
 python-versions = ">=3.7"
 files = [
     {file = "jinja2-3.1.4-py3-none-any.whl", hash = "sha256:bc5dd2abb727a5319567b7a813e6a2e7318c39f4f487cfe6c89c6f9c7d25197d"},
@@ -1054,7 +1013,7 @@ i18n = ["Babel (>=2.7)"]
 name = "joblib"
 version = "1.4.2"
 description = "Lightweight pipelining with Python functions"
-optional = false
+optional = true
 python-versions = ">=3.8"
 files = [
     {file = "joblib-1.4.2-py3-none-any.whl", hash = "sha256:06d478d5674cbc267e7496a410ee875abd68e4340feff4490bcb7afb88060ae6"},
@@ -1161,6 +1120,30 @@ win32-setctime = {version = ">=1.0.0", markers = "sys_platform == \"win32\""}
 [package.extras]
 dev = ["Sphinx (>=4.1.1)", "black (>=19.10b0)", "colorama (>=0.3.4)", "docutils (==0.16)", "flake8 (>=3.7.7)", "isort (>=5.1.1)", "pytest (>=4.6.2)", "pytest-cov (>=2.7.1)", "sphinx-autobuild (>=0.7.1)", "sphinx-rtd-theme (>=0.4.3)", "tox (>=3.9.0)"]
 
+[[package]]
+name = "markdown-it-py"
+version = "3.0.0"
+description = "Python port of markdown-it. Markdown parsing, done right!"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "markdown-it-py-3.0.0.tar.gz", hash = "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb"},
+    {file = "markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1"},
+]
+
+[package.dependencies]
+mdurl = ">=0.1,<1.0"
+
+[package.extras]
+benchmarking = ["psutil", "pytest", "pytest-benchmark"]
+code-style = ["pre-commit (>=3.0,<4.0)"]
+compare = ["commonmark (>=0.9,<1.0)", "markdown (>=3.4,<4.0)", "mistletoe (>=1.0,<2.0)", "mistune (>=2.0,<3.0)", "panflute (>=2.3,<3.0)"]
+linkify = ["linkify-it-py (>=1,<3)"]
+plugins = ["mdit-py-plugins"]
+profiling = ["gprof2dot"]
+rtd = ["jupyter_sphinx", "mdit-py-plugins", "myst-parser", "pyyaml", "sphinx", "sphinx-copybutton", "sphinx-design", "sphinx_book_theme"]
+testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"]
+
 [[package]]
 name = "markupsafe"
 version = "3.0.2"
@@ -1231,11 +1214,166 @@ files = [
     {file = "markupsafe-3.0.2.tar.gz", hash = "sha256:ee55d3edf80167e48ea11a923c7386f4669df67d7994554387f84e7d8b0a2bf0"},
 ]
 
+[[package]]
+name = "marlin-kernels"
+version = "0.2.0"
+description = "Marlin quantization kernels"
+optional = true
+python-versions = ">=3.7"
+files = [
+    {file = "marlin_kernels-0.2.0+cu123torch2.4-cp310-cp310-linux_x86_64.whl", hash = "sha256:9a5afcf19b0f5917e43353cc19873fb3c4d4d0b924e2a95a37884f9ce208d0bd"},
+]
+
+[package.dependencies]
+torch = "*"
+
+[package.source]
+type = "url"
+url = "https://github.com/danieldk/marlin-kernels/releases/download/v0.2.0/marlin_kernels-0.2.0+cu123torch2.4-cp310-cp310-linux_x86_64.whl"
+
+[[package]]
+name = "marlin-kernels"
+version = "0.2.0"
+description = "Marlin quantization kernels"
+optional = true
+python-versions = ">=3.7"
+files = [
+    {file = "marlin_kernels-0.2.0+cu123torch2.4-cp311-cp311-linux_x86_64.whl", hash = "sha256:1e64fcc7ebadfaffa60091ee9201ae3daaf5c1be3be60c8c054143a3dcb72d5d"},
+]
+
+[package.dependencies]
+torch = "*"
+
+[package.source]
+type = "url"
+url = "https://github.com/danieldk/marlin-kernels/releases/download/v0.2.0/marlin_kernels-0.2.0+cu123torch2.4-cp311-cp311-linux_x86_64.whl"
+
+[[package]]
+name = "marlin-kernels"
+version = "0.2.0"
+description = "Marlin quantization kernels"
+optional = true
+python-versions = ">=3.7"
+files = [
+    {file = "marlin_kernels-0.2.0+cu123torch2.4-cp312-cp312-linux_x86_64.whl", hash = "sha256:e75f3ce9b1c13a4ed43a380d88e1d34d297259452db037ec1973ec33dc2eb78e"},
+]
+
+[package.dependencies]
+torch = "*"
+
+[package.source]
+type = "url"
+url = "https://github.com/danieldk/marlin-kernels/releases/download/v0.2.0/marlin_kernels-0.2.0+cu123torch2.4-cp312-cp312-linux_x86_64.whl"
+
+[[package]]
+name = "marlin-kernels"
+version = "0.2.0"
+description = "Marlin quantization kernels"
+optional = true
+python-versions = ">=3.7"
+files = [
+    {file = "marlin_kernels-0.2.0+cu123torch2.4-cp39-cp39-linux_x86_64.whl", hash = "sha256:2f99a27f70b391887ee6adffeeee7c3f4df7fac37393f9fb16d4cace2b3f6457"},
+]
+
+[package.dependencies]
+torch = "*"
+
+[package.source]
+type = "url"
+url = "https://github.com/danieldk/marlin-kernels/releases/download/v0.2.0/marlin_kernels-0.2.0+cu123torch2.4-cp39-cp39-linux_x86_64.whl"
+
+[[package]]
+name = "mdurl"
+version = "0.1.2"
+description = "Markdown URL utilities"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"},
+    {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"},
+]
+
+[[package]]
+name = "moe-kernels"
+version = "0.4.0"
+description = "MoE kernels"
+optional = true
+python-versions = ">=3.7"
+files = [
+    {file = "moe_kernels-0.4.0+cu123torch2.4-cp310-cp310-linux_x86_64.whl", hash = "sha256:3fc0475bb3b9c09bbf08f6f6e9767d10eaba55b558f67a605fe70ae0cbb5e6a4"},
+]
+
+[package.dependencies]
+nvidia-ml-py = "*"
+torch = "*"
+triton = "*"
+
+[package.source]
+type = "url"
+url = "https://github.com/danieldk/moe-kernels/releases/download/v0.4.0/moe_kernels-0.4.0+cu123torch2.4-cp310-cp310-linux_x86_64.whl"
+
+[[package]]
+name = "moe-kernels"
+version = "0.4.0"
+description = "MoE kernels"
+optional = true
+python-versions = ">=3.7"
+files = [
+    {file = "moe_kernels-0.4.0+cu123torch2.4-cp311-cp311-linux_x86_64.whl", hash = "sha256:8ca72a064ceb84a23a3437cc6e6363907ad41588877f6acb1febc010fc7beb22"},
+]
+
+[package.dependencies]
+nvidia-ml-py = "*"
+torch = "*"
+triton = "*"
+
+[package.source]
+type = "url"
+url = "https://github.com/danieldk/moe-kernels/releases/download/v0.4.0/moe_kernels-0.4.0+cu123torch2.4-cp311-cp311-linux_x86_64.whl"
+
+[[package]]
+name = "moe-kernels"
+version = "0.4.0"
+description = "MoE kernels"
+optional = true
+python-versions = ">=3.7"
+files = [
+    {file = "moe_kernels-0.4.0+cu123torch2.4-cp312-cp312-linux_x86_64.whl", hash = "sha256:d302d6b16bb4905b2312dc68da6a6f51e87d0cd3c4bf1f23d995501162399a8e"},
+]
+
+[package.dependencies]
+nvidia-ml-py = "*"
+torch = "*"
+triton = "*"
+
+[package.source]
+type = "url"
+url = "https://github.com/danieldk/moe-kernels/releases/download/v0.4.0/moe_kernels-0.4.0+cu123torch2.4-cp312-cp312-linux_x86_64.whl"
+
+[[package]]
+name = "moe-kernels"
+version = "0.4.0"
+description = "MoE kernels"
+optional = true
+python-versions = ">=3.7"
+files = [
+    {file = "moe_kernels-0.4.0+cu123torch2.4-cp39-cp39-linux_x86_64.whl", hash = "sha256:6aee3e723efa5113c338b40e6cb20fa62da6c442c65c1a6cc97751d34158a93a"},
+]
+
+[package.dependencies]
+nvidia-ml-py = "*"
+torch = "*"
+triton = "*"
+
+[package.source]
+type = "url"
+url = "https://github.com/danieldk/moe-kernels/releases/download/v0.4.0/moe_kernels-0.4.0+cu123torch2.4-cp39-cp39-linux_x86_64.whl"
+
 [[package]]
 name = "mpmath"
 version = "1.3.0"
 description = "Python library for arbitrary-precision floating-point arithmetic"
-optional = false
+optional = true
 python-versions = "*"
 files = [
     {file = "mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c"},
@@ -1395,7 +1533,7 @@ files = [
 name = "networkx"
 version = "3.2.1"
 description = "Python package for creating and manipulating graphs and networks"
-optional = false
+optional = true
 python-versions = ">=3.9"
 files = [
     {file = "networkx-3.2.1-py3-none-any.whl", hash = "sha256:f18c69adc97877c42332c170849c96cefa91881c99a7cb3e95b7c659ebdc1ec2"},
@@ -1492,7 +1630,7 @@ files = [
 name = "nvidia-cublas-cu12"
 version = "12.1.3.1"
 description = "CUBLAS native runtime libraries"
-optional = false
+optional = true
 python-versions = ">=3"
 files = [
     {file = "nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl", hash = "sha256:ee53ccca76a6fc08fb9701aa95b6ceb242cdaab118c3bb152af4e579af792728"},
@@ -1503,7 +1641,7 @@ files = [
 name = "nvidia-cuda-cupti-cu12"
 version = "12.1.105"
 description = "CUDA profiling tools runtime libs."
-optional = false
+optional = true
 python-versions = ">=3"
 files = [
     {file = "nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl", hash = "sha256:e54fde3983165c624cb79254ae9818a456eb6e87a7fd4d56a2352c24ee542d7e"},
@@ -1514,7 +1652,7 @@ files = [
 name = "nvidia-cuda-nvrtc-cu12"
 version = "12.1.105"
 description = "NVRTC native runtime libraries"
-optional = false
+optional = true
 python-versions = ">=3"
 files = [
     {file = "nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl", hash = "sha256:339b385f50c309763ca65456ec75e17bbefcbbf2893f462cb8b90584cd27a1c2"},
@@ -1525,7 +1663,7 @@ files = [
 name = "nvidia-cuda-runtime-cu12"
 version = "12.1.105"
 description = "CUDA Runtime native Libraries"
-optional = false
+optional = true
 python-versions = ">=3"
 files = [
     {file = "nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl", hash = "sha256:6e258468ddf5796e25f1dc591a31029fa317d97a0a94ed93468fc86301d61e40"},
@@ -1536,7 +1674,7 @@ files = [
 name = "nvidia-cudnn-cu12"
 version = "9.1.0.70"
 description = "cuDNN runtime libraries"
-optional = false
+optional = true
 python-versions = ">=3"
 files = [
     {file = "nvidia_cudnn_cu12-9.1.0.70-py3-none-manylinux2014_x86_64.whl", hash = "sha256:165764f44ef8c61fcdfdfdbe769d687e06374059fbb388b6c89ecb0e28793a6f"},
@@ -1550,7 +1688,7 @@ nvidia-cublas-cu12 = "*"
 name = "nvidia-cufft-cu12"
 version = "11.0.2.54"
 description = "CUFFT native runtime libraries"
-optional = false
+optional = true
 python-versions = ">=3"
 files = [
     {file = "nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl", hash = "sha256:794e3948a1aa71fd817c3775866943936774d1c14e7628c74f6f7417224cdf56"},
@@ -1561,7 +1699,7 @@ files = [
 name = "nvidia-curand-cu12"
 version = "10.3.2.106"
 description = "CURAND native runtime libraries"
-optional = false
+optional = true
 python-versions = ">=3"
 files = [
     {file = "nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl", hash = "sha256:9d264c5036dde4e64f1de8c50ae753237c12e0b1348738169cd0f8a536c0e1e0"},
@@ -1572,7 +1710,7 @@ files = [
 name = "nvidia-cusolver-cu12"
 version = "11.4.5.107"
 description = "CUDA solver native runtime libraries"
-optional = false
+optional = true
 python-versions = ">=3"
 files = [
     {file = "nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl", hash = "sha256:8a7ec542f0412294b15072fa7dab71d31334014a69f953004ea7a118206fe0dd"},
@@ -1588,7 +1726,7 @@ nvidia-nvjitlink-cu12 = "*"
 name = "nvidia-cusparse-cu12"
 version = "12.1.0.106"
 description = "CUSPARSE native runtime libraries"
-optional = false
+optional = true
 python-versions = ">=3"
 files = [
     {file = "nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl", hash = "sha256:f3b50f42cf363f86ab21f720998517a659a48131e8d538dc02f8768237bd884c"},
@@ -1598,11 +1736,22 @@ files = [
 [package.dependencies]
 nvidia-nvjitlink-cu12 = "*"
 
+[[package]]
+name = "nvidia-ml-py"
+version = "12.560.30"
+description = "Python Bindings for the NVIDIA Management Library"
+optional = true
+python-versions = "*"
+files = [
+    {file = "nvidia-ml-py-12.560.30.tar.gz", hash = "sha256:f0254dc7400647680a072ee02509bfd46102b60bdfeca321576d4d4817e7fe97"},
+    {file = "nvidia_ml_py-12.560.30-py3-none-any.whl", hash = "sha256:fea371c94d63e38a611c17bbb85fe400e9c8ddb9e8684a9cd0e47786a4bc3c73"},
+]
+
 [[package]]
 name = "nvidia-nccl-cu12"
 version = "2.20.5"
 description = "NVIDIA Collective Communication Library (NCCL) Runtime"
-optional = false
+optional = true
 python-versions = ">=3"
 files = [
     {file = "nvidia_nccl_cu12-2.20.5-py3-none-manylinux2014_aarch64.whl", hash = "sha256:1fc150d5c3250b170b29410ba682384b14581db722b2531b0d8d33c595f33d01"},
@@ -1613,7 +1762,7 @@ files = [
 name = "nvidia-nvjitlink-cu12"
 version = "12.6.77"
 description = "Nvidia JIT LTO Library"
-optional = false
+optional = true
 python-versions = ">=3"
 files = [
     {file = "nvidia_nvjitlink_cu12-12.6.77-py3-none-manylinux2014_aarch64.whl", hash = "sha256:3bf10d85bb1801e9c894c6e197e44dd137d2a0a9e43f8450e9ad13f2df0dd52d"},
@@ -1625,7 +1774,7 @@ files = [
 name = "nvidia-nvtx-cu12"
 version = "12.1.105"
 description = "NVIDIA Tools Extension"
-optional = false
+optional = true
 python-versions = ">=3"
 files = [
     {file = "nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl", hash = "sha256:dc21cf308ca5691e7c04d962e213f8a4aa9bbfa23d95412f452254c2caeb09e5"},
@@ -1634,87 +1783,97 @@ files = [
 
 [[package]]
 name = "opentelemetry-api"
-version = "1.15.0"
+version = "1.25.0"
 description = "OpenTelemetry Python API"
 optional = false
-python-versions = ">=3.7"
+python-versions = ">=3.8"
 files = [
-    {file = "opentelemetry_api-1.15.0-py3-none-any.whl", hash = "sha256:e6c2d2e42140fd396e96edf75a7ceb11073f4efb4db87565a431cc9d0f93f2e0"},
-    {file = "opentelemetry_api-1.15.0.tar.gz", hash = "sha256:79ab791b4aaad27acc3dc3ba01596db5b5aac2ef75c70622c6038051d6c2cded"},
+    {file = "opentelemetry_api-1.25.0-py3-none-any.whl", hash = "sha256:757fa1aa020a0f8fa139f8959e53dec2051cc26b832e76fa839a6d76ecefd737"},
+    {file = "opentelemetry_api-1.25.0.tar.gz", hash = "sha256:77c4985f62f2614e42ce77ee4c9da5fa5f0bc1e1821085e9a47533a9323ae869"},
 ]
 
 [package.dependencies]
 deprecated = ">=1.2.6"
-setuptools = ">=16.0"
+importlib-metadata = ">=6.0,<=7.1"
 
 [[package]]
 name = "opentelemetry-exporter-otlp"
-version = "1.15.0"
+version = "1.25.0"
 description = "OpenTelemetry Collector Exporters"
 optional = false
-python-versions = ">=3.7"
+python-versions = ">=3.8"
 files = [
-    {file = "opentelemetry_exporter_otlp-1.15.0-py3-none-any.whl", hash = "sha256:79f22748b6a54808a0448093dfa189c8490e729f67c134d4c992533d9393b33e"},
-    {file = "opentelemetry_exporter_otlp-1.15.0.tar.gz", hash = "sha256:4f7c49751d9720e2e726e13b0bb958ccade4e29122c305d92c033da432c8d2c5"},
+    {file = "opentelemetry_exporter_otlp-1.25.0-py3-none-any.whl", hash = "sha256:d67a831757014a3bc3174e4cd629ae1493b7ba8d189e8a007003cacb9f1a6b60"},
+    {file = "opentelemetry_exporter_otlp-1.25.0.tar.gz", hash = "sha256:ce03199c1680a845f82e12c0a6a8f61036048c07ec7a0bd943142aca8fa6ced0"},
 ]
 
 [package.dependencies]
-opentelemetry-exporter-otlp-proto-grpc = "1.15.0"
-opentelemetry-exporter-otlp-proto-http = "1.15.0"
+opentelemetry-exporter-otlp-proto-grpc = "1.25.0"
+opentelemetry-exporter-otlp-proto-http = "1.25.0"
+
+[[package]]
+name = "opentelemetry-exporter-otlp-proto-common"
+version = "1.25.0"
+description = "OpenTelemetry Protobuf encoding"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "opentelemetry_exporter_otlp_proto_common-1.25.0-py3-none-any.whl", hash = "sha256:15637b7d580c2675f70246563363775b4e6de947871e01d0f4e3881d1848d693"},
+    {file = "opentelemetry_exporter_otlp_proto_common-1.25.0.tar.gz", hash = "sha256:c93f4e30da4eee02bacd1e004eb82ce4da143a2f8e15b987a9f603e0a85407d3"},
+]
+
+[package.dependencies]
+opentelemetry-proto = "1.25.0"
 
 [[package]]
 name = "opentelemetry-exporter-otlp-proto-grpc"
-version = "1.15.0"
+version = "1.25.0"
 description = "OpenTelemetry Collector Protobuf over gRPC Exporter"
 optional = false
-python-versions = ">=3.7"
+python-versions = ">=3.8"
 files = [
-    {file = "opentelemetry_exporter_otlp_proto_grpc-1.15.0-py3-none-any.whl", hash = "sha256:c2a5492ba7d140109968135d641d06ce3c5bd73c50665f787526065d57d7fd1d"},
-    {file = "opentelemetry_exporter_otlp_proto_grpc-1.15.0.tar.gz", hash = "sha256:844f2a4bb9bcda34e4eb6fe36765e5031aacb36dc60ed88c90fc246942ea26e7"},
+    {file = "opentelemetry_exporter_otlp_proto_grpc-1.25.0-py3-none-any.whl", hash = "sha256:3131028f0c0a155a64c430ca600fd658e8e37043cb13209f0109db5c1a3e4eb4"},
+    {file = "opentelemetry_exporter_otlp_proto_grpc-1.25.0.tar.gz", hash = "sha256:c0b1661415acec5af87625587efa1ccab68b873745ca0ee96b69bb1042087eac"},
 ]
 
 [package.dependencies]
-backoff = {version = ">=1.10.0,<3.0.0", markers = "python_version >= \"3.7\""}
+deprecated = ">=1.2.6"
 googleapis-common-protos = ">=1.52,<2.0"
 grpcio = ">=1.0.0,<2.0.0"
-opentelemetry-api = ">=1.12,<2.0"
-opentelemetry-proto = "1.15.0"
-opentelemetry-sdk = ">=1.12,<2.0"
-
-[package.extras]
-test = ["pytest-grpc"]
+opentelemetry-api = ">=1.15,<2.0"
+opentelemetry-exporter-otlp-proto-common = "1.25.0"
+opentelemetry-proto = "1.25.0"
+opentelemetry-sdk = ">=1.25.0,<1.26.0"
 
 [[package]]
 name = "opentelemetry-exporter-otlp-proto-http"
-version = "1.15.0"
+version = "1.25.0"
 description = "OpenTelemetry Collector Protobuf over HTTP Exporter"
 optional = false
-python-versions = ">=3.7"
+python-versions = ">=3.8"
 files = [
-    {file = "opentelemetry_exporter_otlp_proto_http-1.15.0-py3-none-any.whl", hash = "sha256:3ec2a02196c8a54bf5cbf7fe623a5238625638e83b6047a983bdf96e2bbb74c0"},
-    {file = "opentelemetry_exporter_otlp_proto_http-1.15.0.tar.gz", hash = "sha256:11b2c814249a49b22f6cca7a06b05701f561d577b747f3660dfd67b6eb9daf9c"},
+    {file = "opentelemetry_exporter_otlp_proto_http-1.25.0-py3-none-any.whl", hash = "sha256:2eca686ee11b27acd28198b3ea5e5863a53d1266b91cda47c839d95d5e0541a6"},
+    {file = "opentelemetry_exporter_otlp_proto_http-1.25.0.tar.gz", hash = "sha256:9f8723859e37c75183ea7afa73a3542f01d0fd274a5b97487ea24cb683d7d684"},
 ]
 
 [package.dependencies]
-backoff = {version = ">=1.10.0,<3.0.0", markers = "python_version >= \"3.7\""}
+deprecated = ">=1.2.6"
 googleapis-common-protos = ">=1.52,<2.0"
-opentelemetry-api = ">=1.12,<2.0"
-opentelemetry-proto = "1.15.0"
-opentelemetry-sdk = ">=1.12,<2.0"
+opentelemetry-api = ">=1.15,<2.0"
+opentelemetry-exporter-otlp-proto-common = "1.25.0"
+opentelemetry-proto = "1.25.0"
+opentelemetry-sdk = ">=1.25.0,<1.26.0"
 requests = ">=2.7,<3.0"
 
-[package.extras]
-test = ["responses (==0.22.0)"]
-
 [[package]]
 name = "opentelemetry-instrumentation"
-version = "0.36b0"
+version = "0.46b0"
 description = "Instrumentation Tools & Auto Instrumentation for OpenTelemetry Python"
 optional = false
-python-versions = ">=3.7"
+python-versions = ">=3.8"
 files = [
-    {file = "opentelemetry_instrumentation-0.36b0-py3-none-any.whl", hash = "sha256:83ba4ae7d5292b5b33e0f851cc5c76d8f91196b9b3527800fc13855c33383ac2"},
-    {file = "opentelemetry_instrumentation-0.36b0.tar.gz", hash = "sha256:e3ddac9b3b93408ef26c8ecbf38f717042977e16381bb4cd329a5b4cf16998cf"},
+    {file = "opentelemetry_instrumentation-0.46b0-py3-none-any.whl", hash = "sha256:89cd721b9c18c014ca848ccd11181e6b3fd3f6c7669e35d59c48dc527408c18b"},
+    {file = "opentelemetry_instrumentation-0.46b0.tar.gz", hash = "sha256:974e0888fb2a1e01c38fbacc9483d024bb1132aad92d6d24e2e5543887a7adda"},
 ]
 
 [package.dependencies]
@@ -1724,35 +1883,33 @@ wrapt = ">=1.0.0,<2.0.0"
 
 [[package]]
 name = "opentelemetry-instrumentation-grpc"
-version = "0.36b0"
+version = "0.46b0"
 description = "OpenTelemetry gRPC instrumentation"
 optional = false
-python-versions = ">=3.7"
+python-versions = ">=3.8"
 files = [
-    {file = "opentelemetry_instrumentation_grpc-0.36b0-py3-none-any.whl", hash = "sha256:eaa246ed2083c97b13bab2555cb9d170e8433230a31476c4cab8a17fa03380a4"},
-    {file = "opentelemetry_instrumentation_grpc-0.36b0.tar.gz", hash = "sha256:dc89447c9eb6ea868970f6c13b4ffdac182cdd5a41dd215a0f5393ca6375be55"},
+    {file = "opentelemetry_instrumentation_grpc-0.46b0-py3-none-any.whl", hash = "sha256:cccfb28db07c28849709f2dcf330237fae0fca9f86971bfce27b28bb9a8b0577"},
+    {file = "opentelemetry_instrumentation_grpc-0.46b0.tar.gz", hash = "sha256:9c5738592cf82672805099826b676d352324b54e03f9ac72a1368ba0605d6ff9"},
 ]
 
 [package.dependencies]
 opentelemetry-api = ">=1.12,<2.0"
-opentelemetry-instrumentation = "0.36b0"
-opentelemetry-sdk = ">=1.12,<2.0"
-opentelemetry-semantic-conventions = "0.36b0"
+opentelemetry-instrumentation = "0.46b0"
+opentelemetry-semantic-conventions = "0.46b0"
 wrapt = ">=1.0.0,<2.0.0"
 
 [package.extras]
 instruments = ["grpcio (>=1.27,<2.0)"]
-test = ["opentelemetry-instrumentation-grpc[instruments]", "opentelemetry-sdk (>=1.12,<2.0)", "opentelemetry-test-utils (==0.36b0)", "protobuf (>=3.13,<4.0)"]
 
 [[package]]
 name = "opentelemetry-proto"
-version = "1.15.0"
+version = "1.25.0"
 description = "OpenTelemetry Python Proto"
 optional = false
-python-versions = ">=3.7"
+python-versions = ">=3.8"
 files = [
-    {file = "opentelemetry_proto-1.15.0-py3-none-any.whl", hash = "sha256:044b6d044b4d10530f250856f933442b8753a17f94ae37c207607f733fb9a844"},
-    {file = "opentelemetry_proto-1.15.0.tar.gz", hash = "sha256:9c4008e40ac8cab359daac283fbe7002c5c29c77ea2674ad5626a249e64e0101"},
+    {file = "opentelemetry_proto-1.25.0-py3-none-any.whl", hash = "sha256:f07e3341c78d835d9b86665903b199893befa5e98866f63d22b00d0b7ca4972f"},
+    {file = "opentelemetry_proto-1.25.0.tar.gz", hash = "sha256:35b6ef9dc4a9f7853ecc5006738ad40443701e52c26099e197895cbda8b815a3"},
 ]
 
 [package.dependencies]
@@ -1760,24 +1917,23 @@ protobuf = ">=3.19,<5.0"
 
 [[package]]
 name = "opentelemetry-sdk"
-version = "1.15.0"
+version = "1.25.0"
 description = "OpenTelemetry Python SDK"
 optional = false
-python-versions = ">=3.7"
+python-versions = ">=3.8"
 files = [
-    {file = "opentelemetry_sdk-1.15.0-py3-none-any.whl", hash = "sha256:555c533e9837766119bbccc7a80458c9971d853a6f1da683a2246cd5e53b4645"},
-    {file = "opentelemetry_sdk-1.15.0.tar.gz", hash = "sha256:98dbffcfeebcbff12c0c974292d6ea603180a145904cf838b1fe4d5c99078425"},
+    {file = "opentelemetry_sdk-1.25.0-py3-none-any.whl", hash = "sha256:d97ff7ec4b351692e9d5a15af570c693b8715ad78b8aafbec5c7100fe966b4c9"},
+    {file = "opentelemetry_sdk-1.25.0.tar.gz", hash = "sha256:ce7fc319c57707ef5bf8b74fb9f8ebdb8bfafbe11898410e0d2a761d08a98ec7"},
 ]
 
 [package.dependencies]
-opentelemetry-api = "1.15.0"
-opentelemetry-semantic-conventions = "0.36b0"
-setuptools = ">=16.0"
+opentelemetry-api = "1.25.0"
+opentelemetry-semantic-conventions = "0.46b0"
 typing-extensions = ">=3.7.4"
 
 [[package]]
 name = "opentelemetry-semantic-conventions"
-version = "0.36b0"
+version = "0.46b0"
 description = "OpenTelemetry Semantic Conventions"
 optional = false
 python-versions = ">=3.7"
@@ -1858,13 +2014,13 @@ tests = ["GitPython", "datasets", "optuna", "parameterized", "peft", "psutil", "
 
 [[package]]
 name = "outlines"
-version = "0.0.36"
+version = "0.0.34"
 description = "Probabilistic Generative Model Programming"
 optional = true
 python-versions = ">=3.8"
 files = [
-    {file = "outlines-0.0.36-py3-none-any.whl", hash = "sha256:afa02ca5c449c47731fa06af66d13c2f5ee8b30f8b82b4db90e08215d6f111d1"},
-    {file = "outlines-0.0.36.tar.gz", hash = "sha256:3cffb43143548cd78c6061990feb461cffd5479999391b8390471ea839c2d46e"},
+    {file = "outlines-0.0.34-py3-none-any.whl", hash = "sha256:911588a7e64a4f193b97fb4c501d98ccfd4e95a98f6a3ada67a280bf0c373c50"},
+    {file = "outlines-0.0.34.tar.gz", hash = "sha256:594e7204c770b47a62eb5c2ba7d25ea0ab2e16882b5f04556712a0228d3d3309"},
 ]
 
 [package.dependencies]
@@ -1887,7 +2043,7 @@ transformers = "*"
 
 [package.extras]
 serve = ["fastapi", "pydantic (>=2.0)", "ray (==2.9.0)", "uvicorn", "vllm (>=0.3.0)"]
-test = ["accelerate", "beartype (<0.16.0)", "coverage[toml] (>=5.1)", "datasets", "diff-cover", "huggingface-hub", "llama-cpp-python", "openai (>=1.0.0)", "pre-commit", "pytest", "pytest-benchmark", "pytest-cov", "pytest-mock", "responses", "transformers"]
+test = ["accelerate", "beartype (<0.16.0)", "coverage[toml] (>=5.1)", "datasets", "diff-cover", "huggingface-hub", "llama-cpp-python (>=0.2.42)", "pre-commit", "pytest", "pytest-benchmark", "pytest-cov", "pytest-mock", "responses", "transformers"]
 
 [[package]]
 name = "packaging"
@@ -1904,7 +2060,7 @@ files = [
 name = "pandas"
 version = "2.2.3"
 description = "Powerful data structures for data analysis, time series, and statistics"
-optional = false
+optional = true
 python-versions = ">=3.9"
 files = [
     {file = "pandas-2.2.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1948ddde24197a0f7add2bdc4ca83bf2b1ef84a1bc8ccffd95eda17fd836ecb5"},
@@ -1990,7 +2146,7 @@ xml = ["lxml (>=4.9.2)"]
 name = "peft"
 version = "0.10.0"
 description = "Parameter-Efficient Fine-Tuning (PEFT)"
-optional = false
+optional = true
 python-versions = ">=3.8.0"
 files = [
     {file = "peft-0.10.0-py3-none-any.whl", hash = "sha256:d5249c97e818d3e31f92553c73c2953acd0ec12649b8b749afff7152cbc86cbb"},
@@ -2245,40 +2401,29 @@ files = [
 
 [[package]]
 name = "protobuf"
-version = "3.20.3"
-description = "Protocol Buffers"
+version = "4.25.5"
+description = ""
 optional = false
-python-versions = ">=3.7"
+python-versions = ">=3.8"
 files = [
-    {file = "protobuf-3.20.3-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:f4bd856d702e5b0d96a00ec6b307b0f51c1982c2bf9c0052cf9019e9a544ba99"},
-    {file = "protobuf-3.20.3-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:9aae4406ea63d825636cc11ffb34ad3379335803216ee3a856787bcf5ccc751e"},
-    {file = "protobuf-3.20.3-cp310-cp310-win32.whl", hash = "sha256:28545383d61f55b57cf4df63eebd9827754fd2dc25f80c5253f9184235db242c"},
-    {file = "protobuf-3.20.3-cp310-cp310-win_amd64.whl", hash = "sha256:67a3598f0a2dcbc58d02dd1928544e7d88f764b47d4a286202913f0b2801c2e7"},
-    {file = "protobuf-3.20.3-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:899dc660cd599d7352d6f10d83c95df430a38b410c1b66b407a6b29265d66469"},
-    {file = "protobuf-3.20.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:e64857f395505ebf3d2569935506ae0dfc4a15cb80dc25261176c784662cdcc4"},
-    {file = "protobuf-3.20.3-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:d9e4432ff660d67d775c66ac42a67cf2453c27cb4d738fc22cb53b5d84c135d4"},
-    {file = "protobuf-3.20.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:74480f79a023f90dc6e18febbf7b8bac7508420f2006fabd512013c0c238f454"},
-    {file = "protobuf-3.20.3-cp37-cp37m-win32.whl", hash = "sha256:b6cc7ba72a8850621bfec987cb72623e703b7fe2b9127a161ce61e61558ad905"},
-    {file = "protobuf-3.20.3-cp37-cp37m-win_amd64.whl", hash = "sha256:8c0c984a1b8fef4086329ff8dd19ac77576b384079247c770f29cc8ce3afa06c"},
-    {file = "protobuf-3.20.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:de78575669dddf6099a8a0f46a27e82a1783c557ccc38ee620ed8cc96d3be7d7"},
-    {file = "protobuf-3.20.3-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:f4c42102bc82a51108e449cbb32b19b180022941c727bac0cfd50170341f16ee"},
-    {file = "protobuf-3.20.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:44246bab5dd4b7fbd3c0c80b6f16686808fab0e4aca819ade6e8d294a29c7050"},
-    {file = "protobuf-3.20.3-cp38-cp38-win32.whl", hash = "sha256:c02ce36ec760252242a33967d51c289fd0e1c0e6e5cc9397e2279177716add86"},
-    {file = "protobuf-3.20.3-cp38-cp38-win_amd64.whl", hash = "sha256:447d43819997825d4e71bf5769d869b968ce96848b6479397e29fc24c4a5dfe9"},
-    {file = "protobuf-3.20.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:398a9e0c3eaceb34ec1aee71894ca3299605fa8e761544934378bbc6c97de23b"},
-    {file = "protobuf-3.20.3-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:bf01b5720be110540be4286e791db73f84a2b721072a3711efff6c324cdf074b"},
-    {file = "protobuf-3.20.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:daa564862dd0d39c00f8086f88700fdbe8bc717e993a21e90711acfed02f2402"},
-    {file = "protobuf-3.20.3-cp39-cp39-win32.whl", hash = "sha256:819559cafa1a373b7096a482b504ae8a857c89593cf3a25af743ac9ecbd23480"},
-    {file = "protobuf-3.20.3-cp39-cp39-win_amd64.whl", hash = "sha256:03038ac1cfbc41aa21f6afcbcd357281d7521b4157926f30ebecc8d4ea59dcb7"},
-    {file = "protobuf-3.20.3-py2.py3-none-any.whl", hash = "sha256:a7ca6d488aa8ff7f329d4c545b2dbad8ac31464f1d8b1c87ad1346717731e4db"},
-    {file = "protobuf-3.20.3.tar.gz", hash = "sha256:2e3427429c9cffebf259491be0af70189607f365c2f41c7c3764af6f337105f2"},
+    {file = "protobuf-4.25.5-cp310-abi3-win32.whl", hash = "sha256:5e61fd921603f58d2f5acb2806a929b4675f8874ff5f330b7d6f7e2e784bbcd8"},
+    {file = "protobuf-4.25.5-cp310-abi3-win_amd64.whl", hash = "sha256:4be0571adcbe712b282a330c6e89eae24281344429ae95c6d85e79e84780f5ea"},
+    {file = "protobuf-4.25.5-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:b2fde3d805354df675ea4c7c6338c1aecd254dfc9925e88c6d31a2bcb97eb173"},
+    {file = "protobuf-4.25.5-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:919ad92d9b0310070f8356c24b855c98df2b8bd207ebc1c0c6fcc9ab1e007f3d"},
+    {file = "protobuf-4.25.5-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:fe14e16c22be926d3abfcb500e60cab068baf10b542b8c858fa27e098123e331"},
+    {file = "protobuf-4.25.5-cp38-cp38-win32.whl", hash = "sha256:98d8d8aa50de6a2747efd9cceba361c9034050ecce3e09136f90de37ddba66e1"},
+    {file = "protobuf-4.25.5-cp38-cp38-win_amd64.whl", hash = "sha256:b0234dd5a03049e4ddd94b93400b67803c823cfc405689688f59b34e0742381a"},
+    {file = "protobuf-4.25.5-cp39-cp39-win32.whl", hash = "sha256:abe32aad8561aa7cc94fc7ba4fdef646e576983edb94a73381b03c53728a626f"},
+    {file = "protobuf-4.25.5-cp39-cp39-win_amd64.whl", hash = "sha256:7a183f592dc80aa7c8da7ad9e55091c4ffc9497b3054452d629bb85fa27c2a45"},
+    {file = "protobuf-4.25.5-py3-none-any.whl", hash = "sha256:0aebecb809cae990f8129ada5ca273d9d670b76d9bfc9b1809f0a9c02b7dbf41"},
+    {file = "protobuf-4.25.5.tar.gz", hash = "sha256:7f8249476b4a9473645db7f8ab42b02fe1488cbe5fb72fddd445e0665afd8584"},
 ]
 
 [[package]]
 name = "psutil"
 version = "6.1.0"
 description = "Cross-platform lib for process and system monitoring in Python."
-optional = false
+optional = true
 python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7"
 files = [
     {file = "psutil-6.1.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:ff34df86226c0227c52f38b919213157588a678d049688eded74c76c8ba4a5d0"},
@@ -2319,7 +2464,7 @@ files = [
 name = "pyarrow"
 version = "17.0.0"
 description = "Python library for Apache Arrow"
-optional = false
+optional = true
 python-versions = ">=3.8"
 files = [
     {file = "pyarrow-17.0.0-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:a5c8b238d47e48812ee577ee20c9a2779e6a5904f1708ae240f53ecbee7c9f07"},
@@ -2527,7 +2672,7 @@ testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "no
 name = "python-dateutil"
 version = "2.9.0.post0"
 description = "Extensions to the standard Python datetime module"
-optional = false
+optional = true
 python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7"
 files = [
     {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"},
@@ -2541,7 +2686,7 @@ six = ">=1.5"
 name = "pytz"
 version = "2024.2"
 description = "World timezone definitions, modern and historical"
-optional = false
+optional = true
 python-versions = "*"
 files = [
     {file = "pytz-2024.2-py2.py3-none-any.whl", hash = "sha256:31c7c1817eb7fae7ca4b8c7ee50c72f93aa2dd863de768e1ef4245d426aa0725"},
@@ -2749,6 +2894,24 @@ urllib3 = ">=1.21.1,<3"
 socks = ["PySocks (>=1.5.6,!=1.5.7)"]
 use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"]
 
+[[package]]
+name = "rich"
+version = "13.8.1"
+description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal"
+optional = false
+python-versions = ">=3.7.0"
+files = [
+    {file = "rich-13.8.1-py3-none-any.whl", hash = "sha256:1760a3c0848469b97b558fc61c85233e3dafb69c7a071b4d60c38099d3cd4c06"},
+    {file = "rich-13.8.1.tar.gz", hash = "sha256:8260cda28e3db6bf04d2d1ef4dbc03ba80a824c88b0e7668a0f23126a424844a"},
+]
+
+[package.dependencies]
+markdown-it-py = ">=2.2.0"
+pygments = ">=2.13.0,<3.0.0"
+
+[package.extras]
+jupyter = ["ipywidgets (>=7.5.1,<9)"]
+
 [[package]]
 name = "rpds-py"
 version = "0.20.0"
@@ -3085,85 +3248,66 @@ dev = ["cython-lint (>=0.12.2)", "doit (>=0.36.0)", "mypy", "pycodestyle", "pyde
 doc = ["jupyterlite-pyodide-kernel", "jupyterlite-sphinx (>=0.12.0)", "jupytext", "matplotlib (>=3.5)", "myst-nb", "numpydoc", "pooch", "pydata-sphinx-theme (>=0.15.2)", "sphinx (>=5.0.0)", "sphinx-design (>=0.4.0)"]
 test = ["array-api-strict", "asv", "gmpy2", "hypothesis (>=6.30)", "mpmath", "pooch", "pytest", "pytest-cov", "pytest-timeout", "pytest-xdist", "scikit-umfpack", "threadpoolctl"]
 
-[[package]]
-name = "sentence-transformers"
-version = "3.0.1"
-description = "Multilingual text embeddings"
-optional = false
-python-versions = ">=3.8.0"
-files = [
-    {file = "sentence_transformers-3.0.1-py3-none-any.whl", hash = "sha256:01050cc4053c49b9f5b78f6980b5a72db3fd3a0abb9169b1792ac83875505ee6"},
-    {file = "sentence_transformers-3.0.1.tar.gz", hash = "sha256:8a3d2c537cc4d1014ccc20ac92be3d6135420a3bc60ae29a3a8a9b4bb35fbff6"},
-]
-
-[package.dependencies]
-accelerate = {version = ">=0.20.3", optional = true, markers = "extra == \"train\""}
-datasets = {version = "*", optional = true, markers = "extra == \"train\""}
-huggingface-hub = ">=0.15.1"
-numpy = "*"
-Pillow = "*"
-scikit-learn = "*"
-scipy = "*"
-torch = ">=1.11.0"
-tqdm = "*"
-transformers = ">=4.34.0,<5.0.0"
-
-[package.extras]
-dev = ["accelerate (>=0.20.3)", "datasets", "pre-commit", "pytest", "ruff (>=0.3.0)"]
-train = ["accelerate (>=0.20.3)", "datasets"]
-
 [[package]]
 name = "sentencepiece"
-version = "0.1.99"
+version = "0.2.0"
 description = "SentencePiece python wrapper"
 optional = false
 python-versions = "*"
 files = [
-    {file = "sentencepiece-0.1.99-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0eb528e70571b7c02723e5804322469b82fe7ea418c96051d0286c0fa028db73"},
-    {file = "sentencepiece-0.1.99-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:77d7fafb2c4e4659cbdf303929503f37a26eabc4ff31d3a79bf1c5a1b338caa7"},
-    {file = "sentencepiece-0.1.99-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:be9cf5b9e404c245aeb3d3723c737ba7a8f5d4ba262ef233a431fa6c45f732a0"},
-    {file = "sentencepiece-0.1.99-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:baed1a26464998f9710d20e52607c29ffd4293e7c71c6a1f83f51ad0911ec12c"},
-    {file = "sentencepiece-0.1.99-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9832f08bb372d4c8b567612f8eab9e36e268dff645f1c28f9f8e851be705f6d1"},
-    {file = "sentencepiece-0.1.99-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:019e7535108e309dae2b253a75834fc3128240aa87c00eb80732078cdc182588"},
-    {file = "sentencepiece-0.1.99-cp310-cp310-win32.whl", hash = "sha256:fa16a830416bb823fa2a52cbdd474d1f7f3bba527fd2304fb4b140dad31bb9bc"},
-    {file = "sentencepiece-0.1.99-cp310-cp310-win_amd64.whl", hash = "sha256:14b0eccb7b641d4591c3e12ae44cab537d68352e4d3b6424944f0c447d2348d5"},
-    {file = "sentencepiece-0.1.99-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6d3c56f24183a1e8bd61043ff2c58dfecdc68a5dd8955dc13bab83afd5f76b81"},
-    {file = "sentencepiece-0.1.99-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ed6ea1819fd612c989999e44a51bf556d0ef6abfb553080b9be3d347e18bcfb7"},
-    {file = "sentencepiece-0.1.99-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a2a0260cd1fb7bd8b4d4f39dc2444a8d5fd4e0a0c4d5c899810ef1abf99b2d45"},
-    {file = "sentencepiece-0.1.99-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8a1abff4d1ff81c77cac3cc6fefa34fa4b8b371e5ee51cb7e8d1ebc996d05983"},
-    {file = "sentencepiece-0.1.99-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:004e6a621d4bc88978eecb6ea7959264239a17b70f2cbc348033d8195c9808ec"},
-    {file = "sentencepiece-0.1.99-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db361e03342c41680afae5807590bc88aa0e17cfd1a42696a160e4005fcda03b"},
-    {file = "sentencepiece-0.1.99-cp311-cp311-win32.whl", hash = "sha256:2d95e19168875b70df62916eb55428a0cbcb834ac51d5a7e664eda74def9e1e0"},
-    {file = "sentencepiece-0.1.99-cp311-cp311-win_amd64.whl", hash = "sha256:f90d73a6f81248a909f55d8e6ef56fec32d559e1e9af045f0b0322637cb8e5c7"},
-    {file = "sentencepiece-0.1.99-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:62e24c81e74bd87a6e0d63c51beb6527e4c0add67e1a17bac18bcd2076afcfeb"},
-    {file = "sentencepiece-0.1.99-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:57efcc2d51caff20d9573567d9fd3f854d9efe613ed58a439c78c9f93101384a"},
-    {file = "sentencepiece-0.1.99-cp36-cp36m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6a904c46197993bd1e95b93a6e373dca2f170379d64441041e2e628ad4afb16f"},
-    {file = "sentencepiece-0.1.99-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d89adf59854741c0d465f0e1525b388c0d174f611cc04af54153c5c4f36088c4"},
-    {file = "sentencepiece-0.1.99-cp36-cp36m-win32.whl", hash = "sha256:47c378146928690d1bc106fdf0da768cebd03b65dd8405aa3dd88f9c81e35dba"},
-    {file = "sentencepiece-0.1.99-cp36-cp36m-win_amd64.whl", hash = "sha256:9ba142e7a90dd6d823c44f9870abdad45e6c63958eb60fe44cca6828d3b69da2"},
-    {file = "sentencepiece-0.1.99-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b7b1a9ae4d7c6f1f867e63370cca25cc17b6f4886729595b885ee07a58d3cec3"},
-    {file = "sentencepiece-0.1.99-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d0f644c9d4d35c096a538507b2163e6191512460035bf51358794a78515b74f7"},
-    {file = "sentencepiece-0.1.99-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c8843d23a0f686d85e569bd6dcd0dd0e0cbc03731e63497ca6d5bacd18df8b85"},
-    {file = "sentencepiece-0.1.99-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:33e6f690a1caebb4867a2e367afa1918ad35be257ecdb3455d2bbd787936f155"},
-    {file = "sentencepiece-0.1.99-cp37-cp37m-win32.whl", hash = "sha256:8a321866c2f85da7beac74a824b4ad6ddc2a4c9bccd9382529506d48f744a12c"},
-    {file = "sentencepiece-0.1.99-cp37-cp37m-win_amd64.whl", hash = "sha256:c42f753bcfb7661c122a15b20be7f684b61fc8592c89c870adf52382ea72262d"},
-    {file = "sentencepiece-0.1.99-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:85b476406da69c70586f0bb682fcca4c9b40e5059814f2db92303ea4585c650c"},
-    {file = "sentencepiece-0.1.99-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:cfbcfe13c69d3f87b7fcd5da168df7290a6d006329be71f90ba4f56bc77f8561"},
-    {file = "sentencepiece-0.1.99-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:445b0ec381af1cd4eef95243e7180c63d9c384443c16c4c47a28196bd1cda937"},
-    {file = "sentencepiece-0.1.99-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c6890ea0f2b4703f62d0bf27932e35808b1f679bdb05c7eeb3812b935ba02001"},
-    {file = "sentencepiece-0.1.99-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fb71af492b0eefbf9f2501bec97bcd043b6812ab000d119eaf4bd33f9e283d03"},
-    {file = "sentencepiece-0.1.99-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:27b866b5bd3ddd54166bbcbf5c8d7dd2e0b397fac8537991c7f544220b1f67bc"},
-    {file = "sentencepiece-0.1.99-cp38-cp38-win32.whl", hash = "sha256:b133e8a499eac49c581c3c76e9bdd08c338cc1939e441fee6f92c0ccb5f1f8be"},
-    {file = "sentencepiece-0.1.99-cp38-cp38-win_amd64.whl", hash = "sha256:0eaf3591dd0690a87f44f4df129cf8d05d8a4029b5b6709b489b8e27f9a9bcff"},
-    {file = "sentencepiece-0.1.99-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:38efeda9bbfb55052d482a009c6a37e52f42ebffcea9d3a98a61de7aee356a28"},
-    {file = "sentencepiece-0.1.99-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6c030b081dc1e1bcc9fadc314b19b740715d3d566ad73a482da20d7d46fd444c"},
-    {file = "sentencepiece-0.1.99-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:84dbe53e02e4f8a2e45d2ac3e430d5c83182142658e25edd76539b7648928727"},
-    {file = "sentencepiece-0.1.99-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0b0f55d0a0ee1719b4b04221fe0c9f0c3461dc3dabd77a035fa2f4788eb3ef9a"},
-    {file = "sentencepiece-0.1.99-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:18e800f206cd235dc27dc749299e05853a4e4332e8d3dfd81bf13d0e5b9007d9"},
-    {file = "sentencepiece-0.1.99-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ae1c40cda8f9d5b0423cfa98542735c0235e7597d79caf318855cdf971b2280"},
-    {file = "sentencepiece-0.1.99-cp39-cp39-win32.whl", hash = "sha256:c84ce33af12ca222d14a1cdd37bd76a69401e32bc68fe61c67ef6b59402f4ab8"},
-    {file = "sentencepiece-0.1.99-cp39-cp39-win_amd64.whl", hash = "sha256:350e5c74d739973f1c9643edb80f7cc904dc948578bcb1d43c6f2b173e5d18dd"},
-    {file = "sentencepiece-0.1.99.tar.gz", hash = "sha256:189c48f5cb2949288f97ccdb97f0473098d9c3dcf5a3d99d4eabe719ec27297f"},
+    {file = "sentencepiece-0.2.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:188779e1298a1c8b8253c7d3ad729cb0a9891e5cef5e5d07ce4592c54869e227"},
+    {file = "sentencepiece-0.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bed9cf85b296fa2b76fc2547b9cbb691a523864cebaee86304c43a7b4cb1b452"},
+    {file = "sentencepiece-0.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d7b67e724bead13f18db6e1d10b6bbdc454af574d70efbb36f27d90387be1ca3"},
+    {file = "sentencepiece-0.2.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2fde4b08cfe237be4484c6c7c2e2c75fb862cfeab6bd5449ce4caeafd97b767a"},
+    {file = "sentencepiece-0.2.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c378492056202d1c48a4979650981635fd97875a00eabb1f00c6a236b013b5e"},
+    {file = "sentencepiece-0.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1380ce6540a368de2ef6d7e6ba14ba8f3258df650d39ba7d833b79ee68a52040"},
+    {file = "sentencepiece-0.2.0-cp310-cp310-win32.whl", hash = "sha256:a1151d6a6dd4b43e552394aed0edfe9292820272f0194bd56c7c1660a0c06c3d"},
+    {file = "sentencepiece-0.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:d490142b0521ef22bc1085f061d922a2a6666175bb6b42e588ff95c0db6819b2"},
+    {file = "sentencepiece-0.2.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:17982700c4f6dbb55fa3594f3d7e5dd1c8659a274af3738e33c987d2a27c9d5c"},
+    {file = "sentencepiece-0.2.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7c867012c0e8bcd5bdad0f791609101cb5c66acb303ab3270218d6debc68a65e"},
+    {file = "sentencepiece-0.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7fd6071249c74f779c5b27183295b9202f8dedb68034e716784364443879eaa6"},
+    {file = "sentencepiece-0.2.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:27f90c55a65013cbb8f4d7aab0599bf925cde4adc67ae43a0d323677b5a1c6cb"},
+    {file = "sentencepiece-0.2.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b293734059ef656dcd65be62ff771507bea8fed0a711b6733976e1ed3add4553"},
+    {file = "sentencepiece-0.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e58b47f933aca74c6a60a79dcb21d5b9e47416256c795c2d58d55cec27f9551d"},
+    {file = "sentencepiece-0.2.0-cp311-cp311-win32.whl", hash = "sha256:c581258cf346b327c62c4f1cebd32691826306f6a41d8c4bec43b010dee08e75"},
+    {file = "sentencepiece-0.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:0993dbc665f4113017892f1b87c3904a44d0640eda510abcacdfb07f74286d36"},
+    {file = "sentencepiece-0.2.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:ea5f536e32ea8ec96086ee00d7a4a131ce583a1b18d130711707c10e69601cb2"},
+    {file = "sentencepiece-0.2.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:d0cb51f53b6aae3c36bafe41e86167c71af8370a039f542c43b0cce5ef24a68c"},
+    {file = "sentencepiece-0.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3212121805afc58d8b00ab4e7dd1f8f76c203ddb9dc94aa4079618a31cf5da0f"},
+    {file = "sentencepiece-0.2.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2a3149e3066c2a75e0d68a43eb632d7ae728c7925b517f4c05c40f6f7280ce08"},
+    {file = "sentencepiece-0.2.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:632f3594d3e7ac8b367bca204cb3fd05a01d5b21455acd097ea4c0e30e2f63d7"},
+    {file = "sentencepiece-0.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f295105c6bdbb05bd5e1b0cafbd78ff95036f5d3641e7949455a3f4e5e7c3109"},
+    {file = "sentencepiece-0.2.0-cp312-cp312-win32.whl", hash = "sha256:fb89f811e5efd18bab141afc3fea3de141c3f69f3fe9e898f710ae7fe3aab251"},
+    {file = "sentencepiece-0.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:7a673a72aab81fef5ebe755c6e0cc60087d1f3a4700835d40537183c1703a45f"},
+    {file = "sentencepiece-0.2.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:4547683f330289ec4f093027bfeb87f9ef023b2eb6f879fdc4a8187c7e0ffb90"},
+    {file = "sentencepiece-0.2.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7cd6175f7eaec7142d2bf6f6597ce7db4c9ac89acf93fcdb17410c3a8b781eeb"},
+    {file = "sentencepiece-0.2.0-cp36-cp36m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:859ba1acde782609a0910a26a60e16c191a82bf39b5621107552c0cd79fad00f"},
+    {file = "sentencepiece-0.2.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bcbbef6cc277f8f18f36959e305f10b1c620442d75addc79c21d7073ae581b50"},
+    {file = "sentencepiece-0.2.0-cp36-cp36m-win32.whl", hash = "sha256:536b934e244829e3fe6c4f198652cd82da48adb9aa145c9f00889542726dee3d"},
+    {file = "sentencepiece-0.2.0-cp36-cp36m-win_amd64.whl", hash = "sha256:0a91aaa3c769b52440df56fafda683b3aa48e3f2169cf7ee5b8c8454a7f3ae9b"},
+    {file = "sentencepiece-0.2.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:787e480ca4c1d08c9985a7eb1eae4345c107729c99e9b5a9a00f2575fc7d4b4b"},
+    {file = "sentencepiece-0.2.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f4d158189eb2ecffea3a51edf6d25e110b3678ec47f1a40f2d541eafbd8f6250"},
+    {file = "sentencepiece-0.2.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d1e5ca43013e8935f25457a4fca47e315780172c3e821b4b13a890668911c792"},
+    {file = "sentencepiece-0.2.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7140d9e5a74a0908493bb4a13f1f16a401297bd755ada4c707e842fbf6f0f5bf"},
+    {file = "sentencepiece-0.2.0-cp37-cp37m-win32.whl", hash = "sha256:6cf333625234f247ab357b0bd9836638405ea9082e1543d5b8408f014979dcbf"},
+    {file = "sentencepiece-0.2.0-cp37-cp37m-win_amd64.whl", hash = "sha256:ff88712338b01031910e8e61e7239aff3ce8869ee31a47df63cb38aadd591bea"},
+    {file = "sentencepiece-0.2.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:20813a68d4c221b1849c62c30e1281ea81687894d894b8d4a0f4677d9311e0f5"},
+    {file = "sentencepiece-0.2.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:926ef920ae2e8182db31d3f5d081ada57804e3e1d3a8c4ef8b117f9d9fb5a945"},
+    {file = "sentencepiece-0.2.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:89f65f69636b7e9c015b79dff9c9985a9bc7d19ded6f79ef9f1ec920fdd73ecf"},
+    {file = "sentencepiece-0.2.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0f67eae0dbe6f2d7d6ba50a354623d787c99965f068b81e145d53240198021b0"},
+    {file = "sentencepiece-0.2.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:98501e075f35dd1a1d5a20f65be26839fcb1938752ec61539af008a5aa6f510b"},
+    {file = "sentencepiece-0.2.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e3d1d2cc4882e8d6a1adf9d5927d7716f80617fc693385661caff21888972269"},
+    {file = "sentencepiece-0.2.0-cp38-cp38-win32.whl", hash = "sha256:b99a308a2e5e569031ab164b74e6fab0b6f37dfb493c32f7816225f4d411a6dd"},
+    {file = "sentencepiece-0.2.0-cp38-cp38-win_amd64.whl", hash = "sha256:cdb701eec783d3ec86b7cd4c763adad8eaf6b46db37ee1c36e5e6c44b3fe1b5f"},
+    {file = "sentencepiece-0.2.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:1e0f9c4d0a6b0af59b613175f019916e28ade076e21242fd5be24340d8a2f64a"},
+    {file = "sentencepiece-0.2.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:298f21cc1366eb60311aedba3169d30f885c363ddbf44214b0a587d2908141ad"},
+    {file = "sentencepiece-0.2.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3f1ec95aa1e5dab11f37ac7eff190493fd87770f7a8b81ebc9dd768d1a3c8704"},
+    {file = "sentencepiece-0.2.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7b06b70af54daa4b4904cbb90b4eb6d35c9f3252fdc86c9c32d5afd4d30118d8"},
+    {file = "sentencepiece-0.2.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:22e37bac44dd6603388cb598c64ff7a76e41ca774646f21c23aadfbf5a2228ab"},
+    {file = "sentencepiece-0.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0461324897735512a32d222e3d886e24ad6a499761952b6bda2a9ee6e4313ea5"},
+    {file = "sentencepiece-0.2.0-cp39-cp39-win32.whl", hash = "sha256:38aed822fb76435fa1f12185f10465a94ab9e51d5e8a9159e9a540ce926f0ffd"},
+    {file = "sentencepiece-0.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:d8cf876516548b5a1d6ac4745d8b554f5c07891d55da557925e5c13ff0b4e6ad"},
+    {file = "sentencepiece-0.2.0.tar.gz", hash = "sha256:a52c19171daaf2e697dc6cbe67684e0fa341b1248966f6aebb541de654d15843"},
 ]
 
 [[package]]
@@ -3190,7 +3334,7 @@ type = ["importlib-metadata (>=7.0.2)", "jaraco.develop (>=7.21)", "mypy (==1.11
 name = "six"
 version = "1.16.0"
 description = "Python 2 and 3 compatibility utilities"
-optional = false
+optional = true
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*"
 files = [
     {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"},
@@ -3201,7 +3345,7 @@ files = [
 name = "sympy"
 version = "1.13.3"
 description = "Computer algebra system (CAS) in Python"
-optional = false
+optional = true
 python-versions = ">=3.8"
 files = [
     {file = "sympy-1.13.3-py3-none-any.whl", hash = "sha256:54612cf55a62755ee71824ce692986f23c88ffa77207b30c1368eda4a7060f73"},
@@ -3215,14 +3359,14 @@ mpmath = ">=1.1.0,<1.4"
 dev = ["hypothesis (>=6.70.0)", "pytest (>=7.1.0)"]
 
 [[package]]
-name = "threadpoolctl"
-version = "3.5.0"
-description = "threadpoolctl"
-optional = false
-python-versions = ">=3.8"
+name = "texttable"
+version = "1.7.0"
+description = "module to create simple ASCII tables"
+optional = true
+python-versions = "*"
 files = [
-    {file = "threadpoolctl-3.5.0-py3-none-any.whl", hash = "sha256:56c1e26c150397e58c4926da8eeee87533b1e32bef131bd4bf6a2f45f3185467"},
-    {file = "threadpoolctl-3.5.0.tar.gz", hash = "sha256:082433502dd922bf738de0d8bcc4fdcbf0979ff44c42bd40f5af8a282f6fa107"},
+    {file = "texttable-1.7.0-py2.py3-none-any.whl", hash = "sha256:72227d592c82b3d7f672731ae73e4d1f88cd8e2ef5b075a7a7f01a23a3743917"},
+    {file = "texttable-1.7.0.tar.gz", hash = "sha256:2d2068fb55115807d3ac77a4ca68fa48803e84ebb0ee2340f858107a36522638"},
 ]
 
 [[package]]
@@ -3389,7 +3533,6 @@ filelock = "*"
 huggingface-hub = ">=0.23.2,<1.0"
 numpy = ">=1.17"
 packaging = ">=20.0"
-protobuf = {version = "*", optional = true, markers = "extra == \"sentencepiece\""}
 pyyaml = ">=5.1"
 regex = "!=2019.12.17"
 requests = "*"
@@ -3448,7 +3591,7 @@ vision = ["Pillow (>=10.0.1,<=15.0)"]
 name = "triton"
 version = "3.0.0"
 description = "A language and compiler for custom Deep Learning operations"
-optional = false
+optional = true
 python-versions = "*"
 files = [
     {file = "triton-3.0.0-1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e1efef76935b2febc365bfadf74bcb65a6f959a9872e5bddf44cc9e0adce1e1a"},
@@ -3456,6 +3599,11 @@ files = [
     {file = "triton-3.0.0-1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:34e509deb77f1c067d8640725ef00c5cbfcb2052a1a3cb6a6d343841f92624eb"},
     {file = "triton-3.0.0-1-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bcbf3b1c48af6a28011a5c40a5b3b9b5330530c3827716b5fbf6d7adcc1e53e9"},
     {file = "triton-3.0.0-1-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6e5727202f7078c56f91ff13ad0c1abab14a0e7f2c87e91b12b6f64f3e8ae609"},
+    {file = "triton-3.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:39b052da883351fdf6be3d93cedae6db3b8e3988d3b09ed221bccecfa9612230"},
+    {file = "triton-3.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cd34f19a8582af96e6291d4afce25dac08cb2a5d218c599163761e8e0827208e"},
+    {file = "triton-3.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0d5e10de8c011adeb7c878c6ce0dd6073b14367749e34467f1cff2bde1b78253"},
+    {file = "triton-3.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e8903767951bf86ec960b4fe4e21bc970055afc65e9d57e916d79ae3c93665e3"},
+    {file = "triton-3.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:41004fb1ae9a53fcb3e970745feb87f0e3c94c6ce1ba86e95fa3b8537894bef7"},
 ]
 
 [package.dependencies]
@@ -3468,13 +3616,13 @@ tutorials = ["matplotlib", "pandas", "tabulate"]
 
 [[package]]
 name = "typer"
-version = "0.7.0"
+version = "0.6.1"
 description = "Typer, build great CLIs. Easy to code. Based on Python type hints."
 optional = false
 python-versions = ">=3.6"
 files = [
-    {file = "typer-0.7.0-py3-none-any.whl", hash = "sha256:b5e704f4e48ec263de1c0b3a2387cd405a13767d2f907f44c1a08cbad96f606d"},
-    {file = "typer-0.7.0.tar.gz", hash = "sha256:ff797846578a9f2a201b53442aedeb543319466870fbe1c701eab66dd7681165"},
+    {file = "typer-0.6.1-py3-none-any.whl", hash = "sha256:54b19e5df18654070a82f8c2aa1da456a4ac16a2a83e6dcd9f170e291c56338e"},
+    {file = "typer-0.6.1.tar.gz", hash = "sha256:2d5720a5e63f73eaf31edaa15f6ab87f35f0690f8ca233017d7d23d743a91d73"},
 ]
 
 [package.dependencies]
@@ -3483,8 +3631,8 @@ click = ">=7.1.1,<9.0.0"
 [package.extras]
 all = ["colorama (>=0.4.3,<0.5.0)", "rich (>=10.11.0,<13.0.0)", "shellingham (>=1.3.0,<2.0.0)"]
 dev = ["autoflake (>=1.3.1,<2.0.0)", "flake8 (>=3.8.3,<4.0.0)", "pre-commit (>=2.17.0,<3.0.0)"]
-doc = ["cairosvg (>=2.5.2,<3.0.0)", "mdx-include (>=1.4.1,<2.0.0)", "mkdocs (>=1.1.2,<2.0.0)", "mkdocs-material (>=8.1.4,<9.0.0)", "pillow (>=9.3.0,<10.0.0)"]
-test = ["black (>=22.3.0,<23.0.0)", "coverage (>=6.2,<7.0)", "isort (>=5.0.6,<6.0.0)", "mypy (==0.910)", "pytest (>=4.4.0,<8.0.0)", "pytest-cov (>=2.10.0,<5.0.0)", "pytest-sugar (>=0.9.4,<0.10.0)", "pytest-xdist (>=1.32.0,<4.0.0)", "rich (>=10.11.0,<13.0.0)", "shellingham (>=1.3.0,<2.0.0)"]
+doc = ["mdx-include (>=1.4.1,<2.0.0)", "mkdocs (>=1.1.2,<2.0.0)", "mkdocs-material (>=8.1.4,<9.0.0)"]
+test = ["black (>=22.3.0,<23.0.0)", "coverage (>=5.2,<6.0)", "isort (>=5.0.6,<6.0.0)", "mypy (==0.910)", "pytest (>=4.4.0,<5.4.0)", "pytest-cov (>=2.10.0,<3.0.0)", "pytest-sugar (>=0.9.4,<0.10.0)", "pytest-xdist (>=1.32.0,<2.0.0)", "rich (>=10.11.0,<13.0.0)", "shellingham (>=1.3.0,<2.0.0)"]
 
 [[package]]
 name = "typing-extensions"
@@ -3501,7 +3649,7 @@ files = [
 name = "tzdata"
 version = "2024.2"
 description = "Provider of IANA time zone data"
-optional = false
+optional = true
 python-versions = ">=2"
 files = [
     {file = "tzdata-2024.2-py2.py3-none-any.whl", hash = "sha256:a48093786cdcde33cad18c2555e8532f34422074448fbc874186f0abd79565cd"},
@@ -3622,7 +3770,7 @@ files = [
 name = "xxhash"
 version = "3.5.0"
 description = "Python binding for xxHash"
-optional = false
+optional = true
 python-versions = ">=3.7"
 files = [
     {file = "xxhash-3.5.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ece616532c499ee9afbb83078b1b952beffef121d989841f7f4b3dc5ac0fd212"},
diff --git a/server/requirements_cuda.txt b/server/requirements_cuda.txt
index 88fcc4f3..5de75b6b 100644
--- a/server/requirements_cuda.txt
+++ b/server/requirements_cuda.txt
@@ -1,48 +1,54 @@
-backoff==2.2.1 ; python_version >= "3.9" and python_version < "3.13"
-certifi==2024.2.2 ; python_version >= "3.9" and python_version < "3.13"
+certifi==2024.8.30 ; python_version >= "3.9" and python_version < "3.13"
 charset-normalizer==3.3.2 ; python_version >= "3.9" and python_version < "3.13"
 click==8.1.7 ; python_version >= "3.9" and python_version < "3.13"
 colorama==0.4.6 ; python_version >= "3.9" and python_version < "3.13" and (sys_platform == "win32" or platform_system == "Windows")
 deprecated==1.2.14 ; python_version >= "3.9" and python_version < "3.13"
 einops==0.6.1 ; python_version >= "3.9" and python_version < "3.13"
-filelock==3.14.0 ; python_version >= "3.9" and python_version < "3.13"
-fsspec==2024.5.0 ; python_version >= "3.9" and python_version < "3.13"
-googleapis-common-protos==1.63.0 ; python_version >= "3.9" and python_version < "3.13"
+filelock==3.16.1 ; python_version >= "3.9" and python_version < "3.13"
+fsspec==2024.6.1 ; python_version >= "3.9" and python_version < "3.13"
+googleapis-common-protos==1.65.0 ; python_version >= "3.9" and python_version < "3.13"
 grpc-interceptor==0.15.4 ; python_version >= "3.9" and python_version < "3.13"
-grpcio-reflection==1.62.2 ; python_version >= "3.9" and python_version < "3.13"
-grpcio-status==1.62.2 ; python_version >= "3.9" and python_version < "3.13"
-grpcio==1.64.0 ; python_version >= "3.9" and python_version < "3.13"
-hf-transfer==0.1.6 ; python_version >= "3.9" and python_version < "3.13"
-huggingface-hub==0.23.1 ; python_version >= "3.9" and python_version < "3.13"
-idna==3.7 ; python_version >= "3.9" and python_version < "3.13"
+grpcio-reflection==1.62.3 ; python_version >= "3.9" and python_version < "3.13"
+grpcio-status==1.62.3 ; python_version >= "3.9" and python_version < "3.13"
+grpcio==1.66.1 ; python_version >= "3.9" and python_version < "3.13"
+hf-transfer==0.1.8 ; python_version >= "3.9" and python_version < "3.13"
+huggingface-hub==0.23.5 ; python_version >= "3.9" and python_version < "3.13"
+idna==3.10 ; python_version >= "3.9" and python_version < "3.13"
+importlib-metadata==7.1.0 ; python_version >= "3.9" and python_version < "3.13"
 loguru==0.6.0 ; python_version >= "3.9" and python_version < "3.13"
+markdown-it-py==3.0.0 ; python_version >= "3.9" and python_version < "3.13"
+mdurl==0.1.2 ; python_version >= "3.9" and python_version < "3.13"
 numpy==1.26.4 ; python_version >= "3.9" and python_version < "3.13"
-opentelemetry-api==1.15.0 ; python_version >= "3.9" and python_version < "3.13"
-opentelemetry-exporter-otlp-proto-grpc==1.15.0 ; python_version >= "3.9" and python_version < "3.13"
-opentelemetry-exporter-otlp-proto-http==1.15.0 ; python_version >= "3.9" and python_version < "3.13"
-opentelemetry-exporter-otlp==1.15.0 ; python_version >= "3.9" and python_version < "3.13"
-opentelemetry-instrumentation-grpc==0.36b0 ; python_version >= "3.9" and python_version < "3.13"
-opentelemetry-instrumentation==0.36b0 ; python_version >= "3.9" and python_version < "3.13"
-opentelemetry-proto==1.15.0 ; python_version >= "3.9" and python_version < "3.13"
-opentelemetry-sdk==1.15.0 ; python_version >= "3.9" and python_version < "3.13"
-opentelemetry-semantic-conventions==0.36b0 ; python_version >= "3.9" and python_version < "3.13"
-packaging==24.0 ; python_version >= "3.9" and python_version < "3.13"
-pillow==10.3.0 ; python_version >= "3.9" and python_version < "3.13"
+opentelemetry-api==1.25.0 ; python_version >= "3.9" and python_version < "3.13"
+opentelemetry-exporter-otlp-proto-common==1.25.0 ; python_version >= "3.9" and python_version < "3.13"
+opentelemetry-exporter-otlp-proto-grpc==1.25.0 ; python_version >= "3.9" and python_version < "3.13"
+opentelemetry-exporter-otlp-proto-http==1.25.0 ; python_version >= "3.9" and python_version < "3.13"
+opentelemetry-exporter-otlp==1.25.0 ; python_version >= "3.9" and python_version < "3.13"
+opentelemetry-instrumentation-grpc==0.46b0 ; python_version >= "3.9" and python_version < "3.13"
+opentelemetry-instrumentation==0.46b0 ; python_version >= "3.9" and python_version < "3.13"
+opentelemetry-proto==1.25.0 ; python_version >= "3.9" and python_version < "3.13"
+opentelemetry-sdk==1.25.0 ; python_version >= "3.9" and python_version < "3.13"
+opentelemetry-semantic-conventions==0.46b0 ; python_version >= "3.9" and python_version < "3.13"
+packaging==24.1 ; python_version >= "3.9" and python_version < "3.13"
+pillow==10.4.0 ; python_version >= "3.9" and python_version < "3.13"
 prometheus-client==0.20.0 ; python_version >= "3.9" and python_version < "3.13"
-protobuf==4.25.3 ; python_version >= "3.9" and python_version < "3.13"
+protobuf==4.25.5 ; python_version >= "3.9" and python_version < "3.13"
 py-cpuinfo==9.0.0 ; python_version >= "3.9" and python_version < "3.13"
-pyyaml==6.0.1 ; python_version >= "3.9" and python_version < "3.13"
-regex==2024.5.15 ; python_version >= "3.9" and python_version < "3.13"
-requests==2.32.2 ; python_version >= "3.9" and python_version < "3.13"
-safetensors==0.4.3 ; python_version >= "3.9" and python_version < "3.13"
+pygments==2.18.0 ; python_version >= "3.9" and python_version < "3.13"
+pyyaml==6.0.2 ; python_version >= "3.9" and python_version < "3.13"
+regex==2024.9.11 ; python_version >= "3.9" and python_version < "3.13"
+requests==2.32.3 ; python_version >= "3.9" and python_version < "3.13"
+rich==13.8.1 ; python_version >= "3.9" and python_version < "3.13"
+safetensors==0.4.5 ; python_version >= "3.9" and python_version < "3.13"
 scipy==1.13.1 ; python_version >= "3.9" and python_version < "3.13"
-sentencepiece==0.1.99 ; python_version >= "3.9" and python_version < "3.13"
-setuptools==70.0.0 ; python_version >= "3.9" and python_version < "3.13"
-tokenizers==0.19.1 ; python_version >= "3.9" and python_version < "3.13"
-tqdm==4.66.4 ; python_version >= "3.9" and python_version < "3.13"
-transformers==4.41.1 ; python_version >= "3.9" and python_version < "3.13"
+sentencepiece==0.2.0 ; python_version >= "3.9" and python_version < "3.13"
+setuptools==75.1.0 ; python_version >= "3.9" and python_version < "3.13"
+tokenizers==0.20.0 ; python_version >= "3.9" and python_version < "3.13"
+tqdm==4.66.5 ; python_version >= "3.9" and python_version < "3.13"
+transformers==4.45.0 ; python_version >= "3.9" and python_version < "3.13"
 typer==0.6.1 ; python_version >= "3.9" and python_version < "3.13"
-typing-extensions==4.12.0 ; python_version >= "3.9" and python_version < "3.13"
-urllib3==2.2.1 ; python_version >= "3.9" and python_version < "3.13"
+typing-extensions==4.12.2 ; python_version >= "3.9" and python_version < "3.13"
+urllib3==2.2.3 ; python_version >= "3.9" and python_version < "3.13"
 win32-setctime==1.1.0 ; python_version >= "3.9" and python_version < "3.13" and sys_platform == "win32"
 wrapt==1.16.0 ; python_version >= "3.9" and python_version < "3.13"
+zipp==3.20.2 ; python_version >= "3.9" and python_version < "3.13"
diff --git a/server/requirements_intel.txt b/server/requirements_intel.txt
new file mode 100644
index 00000000..5de75b6b
--- /dev/null
+++ b/server/requirements_intel.txt
@@ -0,0 +1,54 @@
+certifi==2024.8.30 ; python_version >= "3.9" and python_version < "3.13"
+charset-normalizer==3.3.2 ; python_version >= "3.9" and python_version < "3.13"
+click==8.1.7 ; python_version >= "3.9" and python_version < "3.13"
+colorama==0.4.6 ; python_version >= "3.9" and python_version < "3.13" and (sys_platform == "win32" or platform_system == "Windows")
+deprecated==1.2.14 ; python_version >= "3.9" and python_version < "3.13"
+einops==0.6.1 ; python_version >= "3.9" and python_version < "3.13"
+filelock==3.16.1 ; python_version >= "3.9" and python_version < "3.13"
+fsspec==2024.6.1 ; python_version >= "3.9" and python_version < "3.13"
+googleapis-common-protos==1.65.0 ; python_version >= "3.9" and python_version < "3.13"
+grpc-interceptor==0.15.4 ; python_version >= "3.9" and python_version < "3.13"
+grpcio-reflection==1.62.3 ; python_version >= "3.9" and python_version < "3.13"
+grpcio-status==1.62.3 ; python_version >= "3.9" and python_version < "3.13"
+grpcio==1.66.1 ; python_version >= "3.9" and python_version < "3.13"
+hf-transfer==0.1.8 ; python_version >= "3.9" and python_version < "3.13"
+huggingface-hub==0.23.5 ; python_version >= "3.9" and python_version < "3.13"
+idna==3.10 ; python_version >= "3.9" and python_version < "3.13"
+importlib-metadata==7.1.0 ; python_version >= "3.9" and python_version < "3.13"
+loguru==0.6.0 ; python_version >= "3.9" and python_version < "3.13"
+markdown-it-py==3.0.0 ; python_version >= "3.9" and python_version < "3.13"
+mdurl==0.1.2 ; python_version >= "3.9" and python_version < "3.13"
+numpy==1.26.4 ; python_version >= "3.9" and python_version < "3.13"
+opentelemetry-api==1.25.0 ; python_version >= "3.9" and python_version < "3.13"
+opentelemetry-exporter-otlp-proto-common==1.25.0 ; python_version >= "3.9" and python_version < "3.13"
+opentelemetry-exporter-otlp-proto-grpc==1.25.0 ; python_version >= "3.9" and python_version < "3.13"
+opentelemetry-exporter-otlp-proto-http==1.25.0 ; python_version >= "3.9" and python_version < "3.13"
+opentelemetry-exporter-otlp==1.25.0 ; python_version >= "3.9" and python_version < "3.13"
+opentelemetry-instrumentation-grpc==0.46b0 ; python_version >= "3.9" and python_version < "3.13"
+opentelemetry-instrumentation==0.46b0 ; python_version >= "3.9" and python_version < "3.13"
+opentelemetry-proto==1.25.0 ; python_version >= "3.9" and python_version < "3.13"
+opentelemetry-sdk==1.25.0 ; python_version >= "3.9" and python_version < "3.13"
+opentelemetry-semantic-conventions==0.46b0 ; python_version >= "3.9" and python_version < "3.13"
+packaging==24.1 ; python_version >= "3.9" and python_version < "3.13"
+pillow==10.4.0 ; python_version >= "3.9" and python_version < "3.13"
+prometheus-client==0.20.0 ; python_version >= "3.9" and python_version < "3.13"
+protobuf==4.25.5 ; python_version >= "3.9" and python_version < "3.13"
+py-cpuinfo==9.0.0 ; python_version >= "3.9" and python_version < "3.13"
+pygments==2.18.0 ; python_version >= "3.9" and python_version < "3.13"
+pyyaml==6.0.2 ; python_version >= "3.9" and python_version < "3.13"
+regex==2024.9.11 ; python_version >= "3.9" and python_version < "3.13"
+requests==2.32.3 ; python_version >= "3.9" and python_version < "3.13"
+rich==13.8.1 ; python_version >= "3.9" and python_version < "3.13"
+safetensors==0.4.5 ; python_version >= "3.9" and python_version < "3.13"
+scipy==1.13.1 ; python_version >= "3.9" and python_version < "3.13"
+sentencepiece==0.2.0 ; python_version >= "3.9" and python_version < "3.13"
+setuptools==75.1.0 ; python_version >= "3.9" and python_version < "3.13"
+tokenizers==0.20.0 ; python_version >= "3.9" and python_version < "3.13"
+tqdm==4.66.5 ; python_version >= "3.9" and python_version < "3.13"
+transformers==4.45.0 ; python_version >= "3.9" and python_version < "3.13"
+typer==0.6.1 ; python_version >= "3.9" and python_version < "3.13"
+typing-extensions==4.12.2 ; python_version >= "3.9" and python_version < "3.13"
+urllib3==2.2.3 ; python_version >= "3.9" and python_version < "3.13"
+win32-setctime==1.1.0 ; python_version >= "3.9" and python_version < "3.13" and sys_platform == "win32"
+wrapt==1.16.0 ; python_version >= "3.9" and python_version < "3.13"
+zipp==3.20.2 ; python_version >= "3.9" and python_version < "3.13"
diff --git a/server/requirements_rocm.txt b/server/requirements_rocm.txt
index 88fcc4f3..5de75b6b 100644
--- a/server/requirements_rocm.txt
+++ b/server/requirements_rocm.txt
@@ -1,48 +1,54 @@
-backoff==2.2.1 ; python_version >= "3.9" and python_version < "3.13"
-certifi==2024.2.2 ; python_version >= "3.9" and python_version < "3.13"
+certifi==2024.8.30 ; python_version >= "3.9" and python_version < "3.13"
 charset-normalizer==3.3.2 ; python_version >= "3.9" and python_version < "3.13"
 click==8.1.7 ; python_version >= "3.9" and python_version < "3.13"
 colorama==0.4.6 ; python_version >= "3.9" and python_version < "3.13" and (sys_platform == "win32" or platform_system == "Windows")
 deprecated==1.2.14 ; python_version >= "3.9" and python_version < "3.13"
 einops==0.6.1 ; python_version >= "3.9" and python_version < "3.13"
-filelock==3.14.0 ; python_version >= "3.9" and python_version < "3.13"
-fsspec==2024.5.0 ; python_version >= "3.9" and python_version < "3.13"
-googleapis-common-protos==1.63.0 ; python_version >= "3.9" and python_version < "3.13"
+filelock==3.16.1 ; python_version >= "3.9" and python_version < "3.13"
+fsspec==2024.6.1 ; python_version >= "3.9" and python_version < "3.13"
+googleapis-common-protos==1.65.0 ; python_version >= "3.9" and python_version < "3.13"
 grpc-interceptor==0.15.4 ; python_version >= "3.9" and python_version < "3.13"
-grpcio-reflection==1.62.2 ; python_version >= "3.9" and python_version < "3.13"
-grpcio-status==1.62.2 ; python_version >= "3.9" and python_version < "3.13"
-grpcio==1.64.0 ; python_version >= "3.9" and python_version < "3.13"
-hf-transfer==0.1.6 ; python_version >= "3.9" and python_version < "3.13"
-huggingface-hub==0.23.1 ; python_version >= "3.9" and python_version < "3.13"
-idna==3.7 ; python_version >= "3.9" and python_version < "3.13"
+grpcio-reflection==1.62.3 ; python_version >= "3.9" and python_version < "3.13"
+grpcio-status==1.62.3 ; python_version >= "3.9" and python_version < "3.13"
+grpcio==1.66.1 ; python_version >= "3.9" and python_version < "3.13"
+hf-transfer==0.1.8 ; python_version >= "3.9" and python_version < "3.13"
+huggingface-hub==0.23.5 ; python_version >= "3.9" and python_version < "3.13"
+idna==3.10 ; python_version >= "3.9" and python_version < "3.13"
+importlib-metadata==7.1.0 ; python_version >= "3.9" and python_version < "3.13"
 loguru==0.6.0 ; python_version >= "3.9" and python_version < "3.13"
+markdown-it-py==3.0.0 ; python_version >= "3.9" and python_version < "3.13"
+mdurl==0.1.2 ; python_version >= "3.9" and python_version < "3.13"
 numpy==1.26.4 ; python_version >= "3.9" and python_version < "3.13"
-opentelemetry-api==1.15.0 ; python_version >= "3.9" and python_version < "3.13"
-opentelemetry-exporter-otlp-proto-grpc==1.15.0 ; python_version >= "3.9" and python_version < "3.13"
-opentelemetry-exporter-otlp-proto-http==1.15.0 ; python_version >= "3.9" and python_version < "3.13"
-opentelemetry-exporter-otlp==1.15.0 ; python_version >= "3.9" and python_version < "3.13"
-opentelemetry-instrumentation-grpc==0.36b0 ; python_version >= "3.9" and python_version < "3.13"
-opentelemetry-instrumentation==0.36b0 ; python_version >= "3.9" and python_version < "3.13"
-opentelemetry-proto==1.15.0 ; python_version >= "3.9" and python_version < "3.13"
-opentelemetry-sdk==1.15.0 ; python_version >= "3.9" and python_version < "3.13"
-opentelemetry-semantic-conventions==0.36b0 ; python_version >= "3.9" and python_version < "3.13"
-packaging==24.0 ; python_version >= "3.9" and python_version < "3.13"
-pillow==10.3.0 ; python_version >= "3.9" and python_version < "3.13"
+opentelemetry-api==1.25.0 ; python_version >= "3.9" and python_version < "3.13"
+opentelemetry-exporter-otlp-proto-common==1.25.0 ; python_version >= "3.9" and python_version < "3.13"
+opentelemetry-exporter-otlp-proto-grpc==1.25.0 ; python_version >= "3.9" and python_version < "3.13"
+opentelemetry-exporter-otlp-proto-http==1.25.0 ; python_version >= "3.9" and python_version < "3.13"
+opentelemetry-exporter-otlp==1.25.0 ; python_version >= "3.9" and python_version < "3.13"
+opentelemetry-instrumentation-grpc==0.46b0 ; python_version >= "3.9" and python_version < "3.13"
+opentelemetry-instrumentation==0.46b0 ; python_version >= "3.9" and python_version < "3.13"
+opentelemetry-proto==1.25.0 ; python_version >= "3.9" and python_version < "3.13"
+opentelemetry-sdk==1.25.0 ; python_version >= "3.9" and python_version < "3.13"
+opentelemetry-semantic-conventions==0.46b0 ; python_version >= "3.9" and python_version < "3.13"
+packaging==24.1 ; python_version >= "3.9" and python_version < "3.13"
+pillow==10.4.0 ; python_version >= "3.9" and python_version < "3.13"
 prometheus-client==0.20.0 ; python_version >= "3.9" and python_version < "3.13"
-protobuf==4.25.3 ; python_version >= "3.9" and python_version < "3.13"
+protobuf==4.25.5 ; python_version >= "3.9" and python_version < "3.13"
 py-cpuinfo==9.0.0 ; python_version >= "3.9" and python_version < "3.13"
-pyyaml==6.0.1 ; python_version >= "3.9" and python_version < "3.13"
-regex==2024.5.15 ; python_version >= "3.9" and python_version < "3.13"
-requests==2.32.2 ; python_version >= "3.9" and python_version < "3.13"
-safetensors==0.4.3 ; python_version >= "3.9" and python_version < "3.13"
+pygments==2.18.0 ; python_version >= "3.9" and python_version < "3.13"
+pyyaml==6.0.2 ; python_version >= "3.9" and python_version < "3.13"
+regex==2024.9.11 ; python_version >= "3.9" and python_version < "3.13"
+requests==2.32.3 ; python_version >= "3.9" and python_version < "3.13"
+rich==13.8.1 ; python_version >= "3.9" and python_version < "3.13"
+safetensors==0.4.5 ; python_version >= "3.9" and python_version < "3.13"
 scipy==1.13.1 ; python_version >= "3.9" and python_version < "3.13"
-sentencepiece==0.1.99 ; python_version >= "3.9" and python_version < "3.13"
-setuptools==70.0.0 ; python_version >= "3.9" and python_version < "3.13"
-tokenizers==0.19.1 ; python_version >= "3.9" and python_version < "3.13"
-tqdm==4.66.4 ; python_version >= "3.9" and python_version < "3.13"
-transformers==4.41.1 ; python_version >= "3.9" and python_version < "3.13"
+sentencepiece==0.2.0 ; python_version >= "3.9" and python_version < "3.13"
+setuptools==75.1.0 ; python_version >= "3.9" and python_version < "3.13"
+tokenizers==0.20.0 ; python_version >= "3.9" and python_version < "3.13"
+tqdm==4.66.5 ; python_version >= "3.9" and python_version < "3.13"
+transformers==4.45.0 ; python_version >= "3.9" and python_version < "3.13"
 typer==0.6.1 ; python_version >= "3.9" and python_version < "3.13"
-typing-extensions==4.12.0 ; python_version >= "3.9" and python_version < "3.13"
-urllib3==2.2.1 ; python_version >= "3.9" and python_version < "3.13"
+typing-extensions==4.12.2 ; python_version >= "3.9" and python_version < "3.13"
+urllib3==2.2.3 ; python_version >= "3.9" and python_version < "3.13"
 win32-setctime==1.1.0 ; python_version >= "3.9" and python_version < "3.13" and sys_platform == "win32"
 wrapt==1.16.0 ; python_version >= "3.9" and python_version < "3.13"
+zipp==3.20.2 ; python_version >= "3.9" and python_version < "3.13"
diff --git a/server/tests/conftest.py b/server/tests/conftest.py
index 16d2c408..d99771f8 100644
--- a/server/tests/conftest.py
+++ b/server/tests/conftest.py
@@ -1,7 +1,10 @@
 import pytest
-
+import os
 from text_generation_server.pb import generate_pb2
 
+os.environ["USE_PREFIX_CACHING"] = "1"
+os.environ["ATTENTION"] = "flashinfer"
+
 
 @pytest.fixture
 def default_pb_parameters():
diff --git a/server/tests/models/test_bloom.py b/server/tests/models/test_bloom.py
index 4b7dde81..dff5239b 100644
--- a/server/tests/models/test_bloom.py
+++ b/server/tests/models/test_bloom.py
@@ -1,5 +1,3 @@
-# Copyright (C) 2024 Habana Labs, Ltd. an Intel Company.
-
 import pytest
 import torch
 
@@ -7,9 +5,12 @@ from copy import copy
 from transformers import AutoTokenizer
 
 from text_generation_server.pb import generate_pb2
-from text_generation_server.models.causal_lm import CausalLMBatch, PAD_SEQUENCE_TO_MULTIPLE_OF
+from text_generation_server.models.causal_lm import CausalLMBatch
 from text_generation_server.utils import weight_hub_files, download_weights
-from text_generation_server.models.bloom import BloomCausalLMBatch, BLOOM
+from text_generation_server.models.bloom import BloomCausalLMBatch, BLOOMSharded
+from text_generation_server.models.custom_modeling.bloom_modeling import (
+    BloomForCausalLM,
+)
 
 
 @pytest.fixture(scope="session")
@@ -18,7 +19,10 @@ def default_bloom():
     revision = "main"
     filenames = weight_hub_files(model_id, revision, ".safetensors")
     download_weights(filenames, model_id, revision)
-    return BLOOM(model_id)
+    return BLOOMSharded(
+        model_id,
+        model_class=BloomForCausalLM,
+    )
 
 
 @pytest.fixture(scope="session")
@@ -31,8 +35,9 @@ def default_pb_request(default_pb_parameters, default_pb_stop_parameters):
     return generate_pb2.Request(
         id=0,
         inputs="Test",
+        input_chunks=generate_pb2.Input(chunks=[generate_pb2.InputChunk(text="Test")]),
         prefill_logprobs=True,
-        truncate=PAD_SEQUENCE_TO_MULTIPLE_OF,
+        truncate=100,
         parameters=default_pb_parameters,
         stopping_parameters=default_pb_stop_parameters,
     )
@@ -46,7 +51,7 @@ def default_pb_batch(default_pb_request):
 @pytest.fixture
 def default_bloom_batch(default_pb_batch, bloom_560m_tokenizer):
     return BloomCausalLMBatch.from_pb(
-        default_pb_batch, bloom_560m_tokenizer, torch.float32, torch.device("hpu")
+        default_pb_batch, bloom_560m_tokenizer, torch.float32, torch.device("cpu")
     )
 
 
@@ -60,7 +65,7 @@ def default_multi_requests_bloom_batch(default_pb_request, bloom_560m_tokenizer)
 
     batch_pb = generate_pb2.Batch(id=0, requests=[req_0, req_1], size=2)
     return BloomCausalLMBatch.from_pb(
-        batch_pb, bloom_560m_tokenizer, torch.float32, torch.device("hpu")
+        batch_pb, bloom_560m_tokenizer, torch.float32, torch.device("cpu")
     )
 
 
@@ -68,29 +73,30 @@ def test_batch_from_pb(default_pb_batch, default_bloom_batch):
     batch = default_bloom_batch
 
     assert batch.batch_id == default_pb_batch.id
-    assert len(batch.requests) == len(default_pb_batch.requests) == default_pb_batch.size
-    for request, pb_request in zip(batch.requests, default_pb_batch.requests):
-        assert request.data == pb_request
+    assert batch.requests == default_pb_batch.requests
 
-    assert batch.input_ids[0][-1] == 3
-    assert batch.input_ids[0][-2] == 10264
-    assert torch.all(batch.input_ids[0][:-2] == 3)
+    assert len(batch.input_ids) == default_pb_batch.size
+    assert batch.input_ids[0][-1] == 10264
+    assert torch.all(batch.input_ids[0][:-1] == 3)
 
-    assert batch.attention_mask[0][-1] == 0
-    assert batch.attention_mask[0][-2] == 1
-    assert torch.all(batch.attention_mask[0][:-2] == 0)
+    assert batch.attention_mask[0][0] == 1
+    assert torch.all(batch.attention_mask[0][1:] == 0)
 
     assert batch.past_key_values is None
 
     assert all(
         [
-            torch.equal(input_ids, request.all_input_ids[:batch.input_length+1, 0])
-            for input_ids, request in zip(batch.input_ids, batch.requests)
+            torch.equal(input_ids, all_input_ids[:, 0])
+            for input_ids, all_input_ids in zip(batch.input_ids, batch.all_input_ids)
         ]
     )
 
+    assert batch.input_lengths == [1]
+
     assert len(batch) == default_pb_batch.size
-    assert batch.max_input_length == batch.input_length == PAD_SEQUENCE_TO_MULTIPLE_OF - 1
+    assert len(batch.next_token_choosers) == len(batch.stopping_criterias) == len(batch)
+
+    assert batch.max_input_length == batch.input_lengths[0]
 
 
 def test_batch_concatenate_no_prefill(default_bloom_batch):
@@ -102,7 +108,6 @@ def test_causal_lm_batch_type(default_bloom):
     assert default_bloom.batch_type == BloomCausalLMBatch
 
 
-@pytest.mark.skip
 def test_causal_lm_generate_token(default_bloom, default_bloom_batch):
     sequence_length = len(default_bloom_batch.all_input_ids[0])
     generations, next_batch, _ = default_bloom.generate_token(default_bloom_batch)
@@ -152,7 +157,6 @@ def test_causal_lm_generate_token(default_bloom, default_bloom_batch):
     assert generations[0].request_id == 0
 
 
-@pytest.mark.skip
 def test_causal_lm_generate_token_completion(default_bloom, default_bloom_batch):
     next_batch = default_bloom_batch
     for _ in range(default_bloom_batch.stopping_criterias[0].max_new_tokens - 1):
@@ -173,7 +177,6 @@ def test_causal_lm_generate_token_completion(default_bloom, default_bloom_batch)
     )
 
 
-@pytest.mark.skip
 def test_causal_lm_generate_token_completion_multi(
     default_bloom, default_multi_requests_bloom_batch
 ):
@@ -224,7 +227,6 @@ def test_causal_lm_generate_token_completion_multi(
     )
 
 
-@pytest.mark.skip
 def test_batch_concatenate(
     default_bloom, default_bloom_batch, default_multi_requests_bloom_batch
 ):
@@ -265,7 +267,7 @@ def test_batch_concatenate(
     assert next_batch.max_input_length == 3
 
     assert next_batch.requests[0] == next_batch_0.requests[0]
-    assert next_batch.requests[1:] == next_batch_1.requests
+    assert next_batch.requests[1:] == list(next_batch_1.requests)
 
     assert next_batch.next_token_choosers[0] == next_batch_0.next_token_choosers[0]
     assert next_batch.next_token_choosers[1:] == next_batch_1.next_token_choosers
diff --git a/server/tests/models/test_causal_lm.py b/server/tests/models/test_causal_lm.py
index 6a0a474a..11ca5efe 100644
--- a/server/tests/models/test_causal_lm.py
+++ b/server/tests/models/test_causal_lm.py
@@ -1,5 +1,3 @@
-# Copyright (C) 2024 Habana Labs, Ltd. an Intel Company.
-
 import pytest
 import torch
 
@@ -7,27 +5,19 @@ from copy import copy
 from transformers import AutoTokenizer
 
 from text_generation_server.pb import generate_pb2
-from text_generation_server.models import get_model
-from text_generation_server.models.causal_lm import (
-    CausalLMBatch,
-    PREFILL_BATCH_BUCKET_SIZE,
-    PAD_SEQUENCE_TO_MULTIPLE_OF,
-    MAX_TOTAL_TOKENS,
-    BATCH_BUCKET_SIZE,
-)
-
-PAD_TOKEN=0
+from text_generation_server.models.causal_lm import CausalLM, CausalLMBatch
 
 
 @pytest.fixture(scope="session")
 def default_causal_lm():
-    return get_model("meta-llama/Llama-2-7b-hf", None, None, None, None)
+    return CausalLM.fallback("gpt2")
 
 
 @pytest.fixture(scope="session")
-def default_tokenizer(default_causal_lm):
-    default_causal_lm.tokenizer.pad_token_id = PAD_TOKEN
-    return default_causal_lm.tokenizer
+def gpt2_tokenizer():
+    tokenizer = AutoTokenizer.from_pretrained("gpt2", padding_side="left")
+    tokenizer.pad_token_id = 50256
+    return tokenizer
 
 
 @pytest.fixture
@@ -35,8 +25,9 @@ def default_pb_request(default_pb_parameters, default_pb_stop_parameters):
     return generate_pb2.Request(
         id=0,
         inputs="Test",
+        input_chunks=generate_pb2.Input(chunks=[generate_pb2.InputChunk(text="Test")]),
         prefill_logprobs=True,
-        truncate=PAD_SEQUENCE_TO_MULTIPLE_OF,
+        truncate=100,
         parameters=default_pb_parameters,
         stopping_parameters=default_pb_stop_parameters,
     )
@@ -48,14 +39,14 @@ def default_pb_batch(default_pb_request):
 
 
 @pytest.fixture
-def default_causal_lm_batch(default_pb_batch, default_tokenizer):
+def default_causal_lm_batch(default_pb_batch, gpt2_tokenizer):
     return CausalLMBatch.from_pb(
-        default_pb_batch, default_tokenizer, torch.float32, torch.device("hpu")
+        default_pb_batch, gpt2_tokenizer, torch.float32, torch.device("cpu")
     )
 
 
 @pytest.fixture
-def default_multi_requests_causal_lm_batch(default_pb_request, default_tokenizer):
+def default_multi_requests_causal_lm_batch(default_pb_request, gpt2_tokenizer):
     req_0 = copy(default_pb_request)
     req_0.id = 1
     req_1 = default_pb_request
@@ -64,7 +55,7 @@ def default_multi_requests_causal_lm_batch(default_pb_request, default_tokenizer
 
     batch_pb = generate_pb2.Batch(id=1, requests=[req_0, req_1], size=2)
     return CausalLMBatch.from_pb(
-        batch_pb, default_tokenizer, torch.float32, torch.device("hpu")
+        batch_pb, gpt2_tokenizer, torch.float32, torch.device("cpu")
     )
 
 
@@ -72,37 +63,30 @@ def test_batch_from_pb(default_pb_batch, default_causal_lm_batch):
     batch = default_causal_lm_batch
 
     assert batch.batch_id == default_pb_batch.id
-    assert len(batch.requests) == len(default_pb_batch.requests)
+    assert batch.requests == default_pb_batch.requests
 
-    for r in range(0,len(default_pb_batch.requests)):
-        assert batch.requests[r].data == default_pb_batch.requests[r]
+    assert len(batch.input_ids) == default_pb_batch.size
+    assert batch.input_ids[0][-1] == 14402
+    assert torch.all(batch.input_ids[0][:-1] == 50256)
 
-    # For Gaudi we are adding padding of multiplication of bucket size
-    size_of_padded_to_bucket = ((default_pb_batch.size + PREFILL_BATCH_BUCKET_SIZE - 1) // PREFILL_BATCH_BUCKET_SIZE) * PREFILL_BATCH_BUCKET_SIZE
-
-    assert len(batch.input_ids) == size_of_padded_to_bucket
-
-    assert batch.input_ids[0][-2] == 4321
-    assert batch.input_ids[0][-3] == 1
-    assert torch.all(batch.input_ids[0][:-3] == PAD_TOKEN)
-    assert batch.input_ids[0][-1] == PAD_TOKEN
-
-    assert batch.attention_mask[0][-1] == 0
-    assert batch.attention_mask[0, -2] == 1
-    assert batch.attention_mask[0, -3] == 1
-    assert torch.all(batch.attention_mask[0, :-3] == 0)
+    assert batch.attention_mask[0, 0] == 1
+    assert torch.all(batch.attention_mask[0, 1:] == 0)
 
     assert batch.past_key_values is None
+
     assert all(
         [
-            torch.equal(input_ids.to('cpu'), request.all_input_ids[:batch.input_length + 1, 0])
-            for input_ids, request in zip(batch.input_ids, batch.requests)
+            torch.equal(input_ids, all_input_ids[:, 0])
+            for input_ids, all_input_ids in zip(batch.input_ids, batch.all_input_ids)
         ]
     )
 
-    assert len(batch) == default_pb_batch.size
+    assert batch.input_lengths == [1]
 
-    assert batch.max_input_length + 1 == default_pb_batch.requests[0].truncate
+    assert len(batch) == default_pb_batch.size
+    assert len(batch.next_token_choosers) == len(batch.stopping_criterias) == len(batch)
+
+    assert batch.max_input_length == batch.input_lengths[0]
 
 
 def test_batch_concatenate_no_prefill(default_causal_lm_batch):
@@ -115,66 +99,73 @@ def test_causal_lm_batch_type(default_causal_lm):
 
 
 def test_causal_lm_generate_token(default_causal_lm, default_causal_lm_batch):
+    sequence_length = len(default_causal_lm_batch.all_input_ids[0])
+    generations, next_batch, _ = default_causal_lm.generate_token(
+        default_causal_lm_batch
+    )
 
-    sequence_length = len(default_causal_lm_batch.requests[0].all_input_ids)
-    generations, next_batch, _ = default_causal_lm.generate_token([default_causal_lm_batch])
-    padding = next_batch.requests[0].stopping_criteria.max_new_tokens
-
+    assert len(generations) == len(next_batch)
     assert isinstance(next_batch, CausalLMBatch)
-    assert len(next_batch.attention_mask[0]) == PAD_SEQUENCE_TO_MULTIPLE_OF
-    assert next_batch.requests[0].all_input_ids[-padding-2] == 4321
 
-    assert torch.all(next_batch.requests[0].all_input_ids[-padding-1:] == PAD_TOKEN)
-    assert torch.all(next_batch.requests[0].all_input_ids[:-padding-3] == PAD_TOKEN)
+    assert len(next_batch.all_input_ids) == len(next_batch)
+    assert len(next_batch.all_input_ids[0]) == sequence_length + 1
+    assert len(next_batch.attention_mask[0]) == 11
+    assert next_batch.all_input_ids[0][-1] == 13
+    assert next_batch.all_input_ids[0][-2] == 14402
+    assert torch.all(next_batch.all_input_ids[0][:-2] == 50256)
 
-    generations, next_batch, _ = default_causal_lm.generate_token([default_causal_lm_batch])
-    assert torch.all(next_batch.attention_mask[0][PAD_SEQUENCE_TO_MULTIPLE_OF-3:PAD_SEQUENCE_TO_MULTIPLE_OF] == 1)
-    assert torch.all(next_batch.attention_mask[0][:PAD_SEQUENCE_TO_MULTIPLE_OF-3] == 0)
-    assert torch.all(next_batch.attention_mask[0][PAD_SEQUENCE_TO_MULTIPLE_OF+1:] == 0)
+    assert torch.all(next_batch.attention_mask[0][0:2] == 1)
+    assert torch.all(next_batch.attention_mask[0][2:] == 0)
 
-    assert next_batch.requests[0].all_input_ids[-padding-2] == 4321
-    assert next_batch.requests[0].all_input_ids[-padding-1] == 292
-    assert torch.all(next_batch.requests[0].all_input_ids[-padding:] == PAD_TOKEN)
-    assert torch.all(next_batch.requests[0].all_input_ids[:-padding-3] == PAD_TOKEN)
+    assert next_batch.input_ids.shape == (len(next_batch), 1)
+    assert next_batch.input_ids[0, 0] == 13
 
-    assert next_batch.input_length == PAD_SEQUENCE_TO_MULTIPLE_OF
-    assert next_batch.max_input_length == next_batch.input_length
+    assert next_batch.input_lengths == [2]
+    assert next_batch.max_input_length == next_batch.input_lengths[0]
 
     assert next_batch.past_key_values is not None
     assert all(
-        [p[0].shape == (BATCH_BUCKET_SIZE, 32, MAX_TOTAL_TOKENS, 128) for p in next_batch.past_key_values]
+        [p[0].shape == (1, 12, sequence_length, 64) for p in next_batch.past_key_values]
     )
     assert all(
-        [p[1].shape == (BATCH_BUCKET_SIZE, 32, MAX_TOTAL_TOKENS, 128) for p in next_batch.past_key_values]
+        [p[1].shape == (1, 12, sequence_length, 64) for p in next_batch.past_key_values]
     )
     assert all([generation.generated_text is None for generation in generations])
-    assert all([len(generation.prefill_tokens) == PAD_SEQUENCE_TO_MULTIPLE_OF-1 for generation in generations])
-    assert all([generation.tokens.token_ids[0] == 292 for generation in generations])
-    assert all([generation.tokens.texts[0] == "ing" for generation in generations])
+    assert all([len(generation.prefill_tokens) == 1 for generation in generations])
+    assert all(
+        [
+            token_id.item() == 13
+            for generation in generations
+            for token_id in generation.tokens.token_ids
+        ]
+    )
+    assert all(
+        [
+            token_text == "."
+            for generation in generations
+            for token_text in generation.tokens.texts
+        ]
+    )
     assert generations[0].request_id == 0
 
 
 def test_causal_lm_generate_token_completion(
     default_causal_lm, default_causal_lm_batch
 ):
-
     next_batch = default_causal_lm_batch
-    generations, next_batch, _ = default_causal_lm.generate_token([next_batch])
-
-    for _ in range(default_causal_lm_batch.requests[0].stopping_criteria.max_new_tokens - 1):
-        generations, next_batch, _ = default_causal_lm.generate_token([next_batch])
+    for _ in range(default_causal_lm_batch.stopping_criterias[0].max_new_tokens - 1):
+        generations, next_batch, _ = default_causal_lm.generate_token(next_batch)
         assert len(generations) == len(next_batch)
 
-    generations, next_batch, _ = default_causal_lm.generate_token([next_batch])
-
+    generations, next_batch, _ = default_causal_lm.generate_token(next_batch)
     assert next_batch is None
 
     assert len(generations) == 1
-    assert generations[0].generated_text.text == "ing the effect of a new method for the detection"
-    assert generations[0].request_id == default_causal_lm_batch.requests[0].data.id
+    assert generations[0].generated_text.text == ".java:784) at net.minecraft."
+    assert generations[0].request_id == default_causal_lm_batch.requests[0].id
     assert (
         generations[0].generated_text.generated_tokens
-        == default_causal_lm_batch.requests[0].stopping_criteria.max_new_tokens
+        == default_causal_lm_batch.stopping_criterias[0].max_new_tokens
     )
 
 
@@ -182,49 +173,51 @@ def test_causal_lm_generate_token_completion_multi(
     default_causal_lm, default_multi_requests_causal_lm_batch
 ):
     next_batch = default_multi_requests_causal_lm_batch
-    generations, next_batch, _ = default_causal_lm.generate_token([next_batch])
 
     for i in range(
-        default_multi_requests_causal_lm_batch.requests[1].stopping_criteria.max_new_tokens - 1
+        default_multi_requests_causal_lm_batch.stopping_criterias[1].max_new_tokens - 1
     ):
-        generations, next_batch, _ = default_causal_lm.generate_token([next_batch])
+        generations, next_batch, _ = default_causal_lm.generate_token(next_batch)
         assert len(generations) == len(next_batch)
 
-    generations, next_batch, _ = default_causal_lm.generate_token([next_batch])
+    generations, next_batch, _ = default_causal_lm.generate_token(next_batch)
     assert next_batch is not None
 
     assert len(generations) == 2
-    assert generations[1].generated_text.text == "ing the effect of a"
+    assert generations[1].generated_text.text == ".java:784)"
     assert (
         generations[1].request_id
-        == default_multi_requests_causal_lm_batch.requests[1].data.id
+        == default_multi_requests_causal_lm_batch.requests[1].id
     )
     assert (
         generations[1].generated_text.generated_tokens
-        == default_multi_requests_causal_lm_batch.requests[1].stopping_criteria.max_new_tokens
+        == default_multi_requests_causal_lm_batch.stopping_criterias[1].max_new_tokens
+    )
+    # Copy stopping_criterias before filtering
+    stopping_criterias = (
+        default_multi_requests_causal_lm_batch.stopping_criterias.copy()
     )
 
-    next_batch = next_batch.filter([next_batch.requests[0].data.id])
+    next_batch = next_batch.filter([next_batch.requests[0].id])
 
     for _ in range(
-        default_multi_requests_causal_lm_batch.requests[0].stopping_criteria.max_new_tokens - default_multi_requests_causal_lm_batch.requests[1].stopping_criteria.max_new_tokens - 1
+        stopping_criterias[0].max_new_tokens - stopping_criterias[1].max_new_tokens - 1
     ):
-        generations, next_batch, _ = default_causal_lm.generate_token([next_batch])
+        generations, next_batch, _ = default_causal_lm.generate_token(next_batch)
         assert len(generations) == len(next_batch)
 
-    generations, next_batch, _ = default_causal_lm.generate_token([next_batch])
-
+    generations, next_batch, _ = default_causal_lm.generate_token(next_batch)
     assert next_batch is None
 
     assert len(generations) == 1
-    assert generations[0].generated_text.text == "ing the effect of a new method for the detection"
+    assert generations[0].generated_text.text == ".java:784) at net.minecraft."
     assert (
         generations[0].request_id
-        == default_multi_requests_causal_lm_batch.requests[0].data.id
+        == default_multi_requests_causal_lm_batch.requests[0].id
     )
     assert (
         generations[0].generated_text.generated_tokens
-        == default_multi_requests_causal_lm_batch.requests[0].stopping_criteria.max_new_tokens
+        == default_multi_requests_causal_lm_batch.stopping_criterias[0].max_new_tokens
     )
 
 
@@ -232,13 +225,11 @@ def test_batch_concatenate(
     default_causal_lm, default_causal_lm_batch, default_multi_requests_causal_lm_batch
 ):
     next_batch_0 = default_causal_lm_batch
-    _, next_batch_0, _ = default_causal_lm.generate_token([next_batch_0])
-    _, next_batch_0, _ = default_causal_lm.generate_token([next_batch_0])
-    _, next_batch_0, _ = default_causal_lm.generate_token([next_batch_0])
+    _, next_batch_0, _ = default_causal_lm.generate_token(next_batch_0)
+    _, next_batch_0, _ = default_causal_lm.generate_token(next_batch_0)
 
     next_batch_1 = default_multi_requests_causal_lm_batch
-    _, next_batch_1, _ = default_causal_lm.generate_token([next_batch_1])
-    _, next_batch_1, _ = default_causal_lm.generate_token([next_batch_1])
+    _, next_batch_1, _ = default_causal_lm.generate_token(next_batch_1)
 
     # Clone past_key_values before concatenating to compare after,
     # because they are removed from the concatenated batches
@@ -251,135 +242,113 @@ def test_batch_concatenate(
 
     next_batch = CausalLMBatch.concatenate([next_batch_0, next_batch_1])
 
-    assert torch.equal(next_batch.requests[0].all_input_ids, next_batch_0.requests[0].all_input_ids)
-    assert torch.equal(next_batch.requests[1].all_input_ids, next_batch_1.requests[0].all_input_ids)
-    assert torch.equal(next_batch.requests[2].all_input_ids, next_batch_1.requests[1].all_input_ids)
-
+    assert torch.equal(next_batch.all_input_ids[0], next_batch_0.all_input_ids[0])
+    assert torch.equal(next_batch.all_input_ids[1], next_batch_1.all_input_ids[0])
+    assert torch.equal(next_batch.all_input_ids[2], next_batch_1.all_input_ids[1])
 
     assert torch.all(
-            next_batch.attention_mask[0:2, -next_batch.right_padding - 3: -next_batch.right_padding] == 1
+        next_batch.attention_mask[0, : -next_batch.padding_right_offset] == 1
     )
     assert torch.all(
-            next_batch.attention_mask[2, -next_batch.right_padding - 4: -next_batch.right_padding] == 1
+        next_batch.attention_mask[1:, 1 : -next_batch.padding_right_offset] == 1
     )
-    assert torch.all(
-            next_batch.attention_mask[3, -next_batch.right_padding - 3: -next_batch.right_padding] == 1
-    )
-
-    assert torch.all(
-            next_batch.attention_mask[0:2, :-next_batch.right_padding-3] == 0)
-    assert torch.all(
-        next_batch.attention_mask[2, :-next_batch.right_padding-4] == 0)
-    assert torch.all(
-        next_batch.attention_mask[3, :-next_batch.right_padding-3] == 0)
+    assert torch.all(next_batch.attention_mask[1:, 3:] == 0)
 
     assert next_batch.batch_id == 0
-    assert next_batch.input_ids[0,-next_batch.right_padding - 3] == 1
-    assert next_batch.input_ids[0,-next_batch.right_padding - 2] == 4321
-    assert next_batch.input_ids[0,-next_batch.right_padding - 1] == 292
+    assert next_batch.input_ids[0, 0] == 12355
+    assert torch.all(next_batch.input_ids[1:] == 13)
 
-    assert next_batch.max_input_length == 129
-
-    assert torch.all(next_batch.input_ids[0,-next_batch.right_padding:] == PAD_TOKEN)
-    assert torch.all(next_batch.input_ids[1,-next_batch.right_padding:] == PAD_TOKEN)
-    assert torch.all(next_batch.input_ids[2,-next_batch.right_padding:] == PAD_TOKEN)
-    assert torch.all(next_batch.input_ids[3,-next_batch.right_padding:] == PAD_TOKEN)
-
-    assert next_batch.input_length == PAD_SEQUENCE_TO_MULTIPLE_OF +1
-    assert next_batch.max_input_length == PAD_SEQUENCE_TO_MULTIPLE_OF +1
+    assert next_batch.input_lengths == [3, 2, 2]
+    assert next_batch.max_input_length == 3
 
     assert next_batch.requests[0] == next_batch_0.requests[0]
-    assert next_batch.requests[1:] == next_batch_1.requests
+    assert next_batch.requests[1:] == list(next_batch_1.requests)
 
-    assert next_batch.requests[0].stopping_criteria == next_batch_0.requests[0].stopping_criteria
-    assert next_batch.requests[1].stopping_criteria == next_batch_1.requests[0].stopping_criteria
-    assert next_batch.requests[2].stopping_criteria == next_batch_1.requests[1].stopping_criteria
-
-    assert next_batch.past_key_values is not None
-
-    assert all([p[0].shape == (8, 32, 2048, 128) for p in next_batch.past_key_values])
-    assert all([p[1].shape == (8, 32, 2048, 128) for p in next_batch.past_key_values])
+    assert next_batch.next_token_choosers[0] == next_batch_0.next_token_choosers[0]
+    assert next_batch.next_token_choosers[1:] == next_batch_1.next_token_choosers
+
+    assert next_batch.stopping_criterias[0] == next_batch_0.stopping_criterias[0]
+    assert next_batch.stopping_criterias[1:] == next_batch_1.stopping_criterias
 
     assert next_batch.past_key_values is not None
+    assert all([p[0].shape == (3, 12, 2, 64) for p in next_batch.past_key_values])
+    assert all([p[1].shape == (3, 12, 2, 64) for p in next_batch.past_key_values])
 
     for i, past in enumerate(next_batch.past_key_values):
-        assert torch.equal(next_batch_0_past_key_values[i][0][0, 0,0:128], past[0][0][0][1:129])
+        assert torch.equal(next_batch_0_past_key_values[i][0][0, :, -2:], past[0][0])
         assert torch.equal(
-            next_batch_1_past_key_values[i][0][:, :, 0:1][0], past[0][1:, :, 1 :2, :][0]
+            next_batch_1_past_key_values[i][0][:, :, -1:], past[0][1:, :, -1:, :]
         )
 
-        assert torch.equal(next_batch_0_past_key_values[i][1][0, 0,0:128], past[1][0][0][1:129])
+        assert torch.equal(next_batch_0_past_key_values[i][1][0, :, -2:], past[1][0])
         assert torch.equal(
-            next_batch_1_past_key_values[i][1][:, :, 0:1][0], past[1][1:, :, 1 :2, :][0]
+            next_batch_1_past_key_values[i][1][:, :, -1:], past[1][1:, :, -1:, :]
         )
 
-    generations, next_batch, _ = default_causal_lm.generate_token([next_batch])
-
     for _ in range(
-        default_multi_requests_causal_lm_batch.requests[1].stopping_criteria.max_new_tokens - 2
+        default_multi_requests_causal_lm_batch.stopping_criterias[1].max_new_tokens - 2
     ):
-        generations, next_batch, _ = default_causal_lm.generate_token([next_batch])
+        generations, next_batch, _ = default_causal_lm.generate_token(next_batch)
         assert len(generations) == len(next_batch)
 
-    generations, next_batch, _ = default_causal_lm.generate_token([next_batch])
+    generations, next_batch, _ = default_causal_lm.generate_token(next_batch)
     assert next_batch is not None
 
     assert len(generations) == 3
-    assert generations[2].generated_text.text == "ing the effect of a"
-
+    assert generations[2].generated_text.text == ".java:784)"
     assert (
         generations[2].request_id
-        == default_multi_requests_causal_lm_batch.requests[1].data.id
+        == default_multi_requests_causal_lm_batch.requests[1].id
     )
     assert (
         generations[2].generated_text.generated_tokens
-        == default_multi_requests_causal_lm_batch.requests[1].stopping_criteria.max_new_tokens
+        == default_multi_requests_causal_lm_batch.stopping_criterias[1].max_new_tokens
     )
 
     next_batch = next_batch.filter(
-        [next_batch.requests[0].data.id, next_batch.requests[1].data.id]
+        [next_batch.requests[0].id, next_batch.requests[1].id]
     )
 
     for _ in range(
-        default_causal_lm_batch.requests[0].stopping_criteria.max_new_tokens
-        - default_multi_requests_causal_lm_batch.requests[1].stopping_criteria.max_new_tokens
+        default_causal_lm_batch.stopping_criterias[0].max_new_tokens
+        - default_multi_requests_causal_lm_batch.stopping_criterias[1].max_new_tokens
         - 2
     ):
-        generations, next_batch, _ = default_causal_lm.generate_token([next_batch])
+        generations, next_batch, _ = default_causal_lm.generate_token(next_batch)
         assert len(generations) == len(next_batch)
 
-    generations, next_batch, _ = default_causal_lm.generate_token([next_batch])
+    generations, next_batch, _ = default_causal_lm.generate_token(next_batch)
     assert next_batch is not None
 
     assert len(generations) == 2
-    assert generations[0].generated_text.text == "ing the effect of a new method for the detection"
-    assert generations[0].request_id == default_causal_lm_batch.requests[0].data.id
+    assert generations[0].generated_text.text == ".java:784) at net.minecraft."
+    assert generations[0].request_id == default_causal_lm_batch.requests[0].id
     assert (
         generations[0].generated_text.generated_tokens
-        == default_causal_lm_batch.requests[0].stopping_criteria.max_new_tokens
+        == default_causal_lm_batch.stopping_criterias[0].max_new_tokens
     )
 
-    next_batch = next_batch.filter([next_batch.requests[1].data.id])
+    next_batch = next_batch.filter([next_batch.requests[1].id])
 
     for _ in range(
-        default_multi_requests_causal_lm_batch.requests[0].stopping_criteria.max_new_tokens
-        - default_causal_lm_batch.requests[0].stopping_criteria.max_new_tokens
-        - default_multi_requests_causal_lm_batch.requests[1].stopping_criteria.max_new_tokens
+        default_multi_requests_causal_lm_batch.stopping_criterias[0].max_new_tokens
+        - default_causal_lm_batch.stopping_criterias[0].max_new_tokens
+        - default_multi_requests_causal_lm_batch.stopping_criterias[1].max_new_tokens
         - 4
     ):
-        generations, next_batch, _ = default_causal_lm.generate_token([next_batch])
+        generations, next_batch, _ = default_causal_lm.generate_token(next_batch)
         assert len(generations) == len(next_batch)
 
-    generations, next_batch, _ = default_causal_lm.generate_token([next_batch])
+    generations, next_batch, _ = default_causal_lm.generate_token(next_batch)
     assert next_batch is None
 
     assert len(generations) == 1
-    assert generations[0].generated_text.text == "ing the effect of a new method for the detection"
+    assert generations[0].generated_text.text == ".java:784) at net.minecraft."
     assert (
         generations[0].request_id
-        == default_multi_requests_causal_lm_batch.requests[0].data.id
+        == default_multi_requests_causal_lm_batch.requests[0].id
     )
     assert (
         generations[0].generated_text.generated_tokens
-        == default_multi_requests_causal_lm_batch.requests[0].stopping_criteria.max_new_tokens
+        == default_multi_requests_causal_lm_batch.stopping_criterias[0].max_new_tokens
     )
diff --git a/server/tests/models/test_grammar.py b/server/tests/models/test_grammar.py
deleted file mode 100644
index b5e65620..00000000
--- a/server/tests/models/test_grammar.py
+++ /dev/null
@@ -1,245 +0,0 @@
-# Copyright (C) 2024 Habana Labs, Ltd. an Intel Company.
-
-import json
-import pytest
-import torch
-
-from copy import copy
-
-from text_generation_server.pb import generate_pb2
-from text_generation_server.models import get_model
-from text_generation_server.models.causal_lm import (
-    CausalLMBatch,
-    PAD_SEQUENCE_TO_MULTIPLE_OF,
-)
-
-PAD_TOKEN=0
-
-
-@pytest.fixture
-def default_pb_grammar_parameters():
-    grammar_schema = {
-        "properties": {
-            "activity": {
-                "type": "string"
-            },
-            "animals": {
-                "items": {
-                    "type":"string"
-                },
-                "type": "array"
-            }
-        },
-        "required": ["activity", "animals"]
-    }
-    return generate_pb2.NextTokenChooserParameters(
-        temperature=1.0,
-        repetition_penalty=1.3,
-        top_k=0,
-        top_p=1.0,
-        typical_p=1.0,
-        do_sample=False,
-        grammar_type=generate_pb2.GrammarType.GRAMMAR_TYPE_JSON,
-        grammar=json.dumps(grammar_schema).encode('utf-8'),
-    )
-
-
-@pytest.fixture(scope="session")
-def default_grammar_response():
-    return [
-        29912, 376, 29874, 312, 2068, 1115, 29871, 13, 29908, 29890,
-        638, 292, 613, 259, 376, 273, 3039, 29879, 1115,518, 1678,
-        376, 26169, 3284, 4117, 3284, 336, 617, 6150, 3108, 500, 2
-    ]
-
-
-@pytest.fixture(scope="session")
-def default_causal_lm():
-    return get_model("meta-llama/Llama-2-7b-hf", None, None, None, None)
-
-
-@pytest.fixture(scope="session")
-def default_tokenizer(default_causal_lm):
-    default_causal_lm.tokenizer.pad_token_id = PAD_TOKEN
-    return default_causal_lm.tokenizer
-
-
-@pytest.fixture
-def default_pb_request(default_pb_parameters):
-    return generate_pb2.Request(
-        id=0,
-        inputs="Test",
-        prefill_logprobs=True,
-        truncate=PAD_SEQUENCE_TO_MULTIPLE_OF,
-        parameters=default_pb_parameters,
-        stopping_parameters=generate_pb2.StoppingCriteriaParameters(stop_sequences=[], max_new_tokens=10),
-    )
-
-
-@pytest.fixture
-def default_pb_grammar_request(default_pb_grammar_parameters):
-    return generate_pb2.Request(
-        id=1,
-        inputs=f"Please use the following JSON schema to generate the output: I saw a puppy a cat and a raccoon during my bike ride in the park",
-        prefill_logprobs=True,
-        truncate=PAD_SEQUENCE_TO_MULTIPLE_OF,
-        parameters=default_pb_grammar_parameters,
-        stopping_parameters=generate_pb2.StoppingCriteriaParameters(stop_sequences=[], max_new_tokens=50),
-    )
-
-
-@pytest.fixture
-def default_pb_batch(default_pb_request):
-    return generate_pb2.Batch(id=0, requests=[default_pb_request], size=1)
-
-
-@pytest.fixture
-def default_pb_grammar_batch(default_pb_grammar_request):
-    return generate_pb2.Batch(id=1, requests=[default_pb_grammar_request], size=1)
-
-
-@pytest.fixture
-def default_causal_lm_batch(default_pb_batch, default_tokenizer):
-    return CausalLMBatch.from_pb(
-        default_pb_batch, default_tokenizer, torch.float32, torch.device("hpu")
-    )
-
-
-@pytest.fixture
-def default_causal_lm_grammar_batch(default_pb_grammar_batch, default_tokenizer):
-    return CausalLMBatch.from_pb(
-        default_pb_grammar_batch, default_tokenizer, torch.float32, torch.device("hpu")
-    )
-
-
-@pytest.fixture
-def default_two_causal_lm_grammar_batches(default_pb_grammar_request, default_tokenizer):
-    req_0 = default_pb_grammar_request
-    req_0.id = 0
-    req_1 = copy(default_pb_grammar_request)
-    req_1.id = 1
-
-    batch_0 = generate_pb2.Batch(id=0, requests=[req_0], size=1)
-    batch_1 = generate_pb2.Batch(id=1, requests=[req_1], size=1)
-    return [
-        CausalLMBatch.from_pb(
-            b, default_tokenizer, torch.float32, torch.device("hpu")
-        ) for b in [batch_0, batch_1]
-    ]
-
-
-def test_single_grammar_batch(
-    default_causal_lm, default_causal_lm_grammar_batch, default_grammar_response
-):
-    counter = 0
-    batch = default_causal_lm_grammar_batch
-
-    # prefill request
-    generations, next_batch, _ = default_causal_lm.generate_token([batch])
-
-    # generate untill done
-    while next_batch is not None:
-        generations, next_batch, _ = default_causal_lm.generate_token([next_batch])
-        assert len(generations) == 1
-        assert generations[0].tokens.token_ids[0] == default_grammar_response[counter]
-        counter += 1
-    print(generations[0].generated_text.text)
-
-
-def test_multi_grammar_batches(
-    default_causal_lm, default_two_causal_lm_grammar_batches, default_grammar_response
-):
-    counter_0, counter_1 = 0, 0
-    batch_0, batch_1 = default_two_causal_lm_grammar_batches
-
-    # prefill first request
-    generations, next_batch, _ = default_causal_lm.generate_token([batch_0])
-    generations, next_batch, _ = default_causal_lm.generate_token([next_batch])
-    assert len(generations) == 1
-    assert generations[0].tokens.token_ids[0] == default_grammar_response[counter_0]
-    counter_0 += 1
-
-    # prefill second request
-    generations, next_batch_1, _ = default_causal_lm.generate_token([batch_1])
-
-    # concatenate and generate
-    generations, next_batch, _ = default_causal_lm.generate_token([next_batch, next_batch_1])
-    assert len(generations) == 2
-    assert generations[0].tokens.token_ids[0] == default_grammar_response[counter_0]
-    assert generations[1].tokens.token_ids[0] == default_grammar_response[counter_1]
-    counter_0 += 1
-    counter_1 += 1
-
-    # generate untill first request is done
-    while generations[0].generated_text is None:
-        generations, next_batch, _ = default_causal_lm.generate_token([next_batch])
-        assert len(generations) == 2
-        assert generations[0].tokens.token_ids[0] == default_grammar_response[counter_0]
-        assert generations[1].tokens.token_ids[0] == default_grammar_response[counter_1]
-        counter_0 += 1
-        counter_1 += 1
-
-    # filter finished request
-    response = generations[0].generated_text.text
-    next_batch = next_batch.filter([next_batch.requests[1].data.id])
-
-    # generate last tokens for second request
-    while next_batch is not None:
-        generations, next_batch, _ = default_causal_lm.generate_token([next_batch])
-        assert len(generations) == 1
-        assert generations[0].tokens.token_ids[0] == default_grammar_response[counter_1]
-        counter_1 += 1
-
-    assert response == generations[0].generated_text.text
-
-
-def test_grammar_and_causal_batch(
-    default_causal_lm, default_causal_lm_grammar_batch, default_causal_lm_batch, default_grammar_response
-):
-    counter = 0
-    generations, next_batch, _ = default_causal_lm.generate_token([default_causal_lm_grammar_batch])
-    generations, next_batch, _ = default_causal_lm.generate_token([next_batch])
-    assert len(generations) == 1
-    assert generations[0].tokens.token_ids[0] == default_grammar_response[counter]
-    counter += 1
-
-    generations, next_batch, _ = default_causal_lm.generate_token([next_batch])
-    assert len(generations) == 1
-    assert generations[0].tokens.token_ids[0] == default_grammar_response[counter]
-    counter += 1
-
-    # prefill second request
-    generations, next_batch_1, _ = default_causal_lm.generate_token([default_causal_lm_batch])
-
-    # concatenate and generate
-    generations, next_batch, _ = default_causal_lm.generate_token([next_batch, next_batch_1])
-    assert len(generations) == 2
-    assert generations[0].tokens.token_ids[0] == default_grammar_response[counter]
-    counter += 1
-
-    # generate untill second request is done
-    for _ in range(
-        next_batch.requests[1].stopping_criteria.max_new_tokens - 1
-    ):
-        generations, next_batch, _ = default_causal_lm.generate_token([next_batch])
-        assert len(generations) == 2
-        assert generations[0].tokens.token_ids[0] == default_grammar_response[counter]
-        counter += 1
-
-    # filter finished request
-    assert len(generations) == 2
-    assert (
-        generations[1].request_id == next_batch.requests[1].data.id
-    )
-    assert (
-        generations[1].generated_text.generated_tokens == next_batch.requests[1].stopping_criteria.max_new_tokens
-    )
-    assert generations[1].generated_text.text == "ing the effect of a new method for the detection"
-    next_batch = next_batch.filter([next_batch.requests[0].data.id])
-
-    # generate untill done
-    while next_batch is not None:
-        generations, next_batch, _ = default_causal_lm.generate_token([next_batch])
-        assert len(generations) == 1
-        assert generations[0].tokens.token_ids[0] == default_grammar_response[counter]
-        counter += 1
diff --git a/server/tests/models/test_model.py b/server/tests/models/test_model.py
index 32bcd45f..8441e8c6 100644
--- a/server/tests/models/test_model.py
+++ b/server/tests/models/test_model.py
@@ -17,7 +17,12 @@ def get_test_model():
     tokenizer = AutoTokenizer.from_pretrained("huggingface/llama-7b")
 
     model = TestModel(
-        torch.nn.Linear(1, 1), tokenizer, False, torch.float32, torch.device("cpu")
+        "test_model_id",
+        torch.nn.Linear(1, 1),
+        tokenizer,
+        False,
+        torch.float32,
+        torch.device("cpu"),
     )
     return model
 
diff --git a/server/tests/models/test_santacoder.py b/server/tests/models/test_santacoder.py
index 1e40e766..d5c91bff 100644
--- a/server/tests/models/test_santacoder.py
+++ b/server/tests/models/test_santacoder.py
@@ -1,13 +1,12 @@
 import pytest
 
 from text_generation_server.pb import generate_pb2
-from text_generation_server.models.causal_lm import CausalLMBatch
-from text_generation_server.models.santacoder import SantaCoder
+from text_generation_server.models.causal_lm import CausalLMBatch, CausalLM
 
 
 @pytest.fixture(scope="session")
 def default_santacoder():
-    return SantaCoder("bigcode/santacoder")
+    return CausalLM.fallback(model_id="bigcode/santacoder")
 
 
 @pytest.fixture
@@ -15,6 +14,7 @@ def default_pb_request(default_pb_parameters, default_pb_stop_parameters):
     return generate_pb2.Request(
         id=0,
         inputs="def",
+        input_chunks=generate_pb2.Input(chunks=[generate_pb2.InputChunk(text="def")]),
         prefill_logprobs=True,
         truncate=100,
         parameters=default_pb_parameters,
@@ -32,6 +32,13 @@ def default_fim_pb_request(default_pb_parameters, default_pb_stop_parameters):
     return generate_pb2.Request(
         id=0,
         inputs="<fim-prefix>def<fim-suffix>world<fim-middle>",
+        input_chunks=generate_pb2.Input(
+            chunks=[
+                generate_pb2.InputChunk(
+                    text="<fim-prefix>def<fim-suffix>world<fim-middle>"
+                )
+            ]
+        ),
         prefill_logprobs=True,
         truncate=100,
         parameters=default_pb_parameters,
diff --git a/server/tests/models/test_seq2seq_lm.py b/server/tests/models/test_seq2seq_lm.py
index ba9f5578..5ba7c64c 100644
--- a/server/tests/models/test_seq2seq_lm.py
+++ b/server/tests/models/test_seq2seq_lm.py
@@ -20,7 +20,7 @@ def mt0_small_tokenizer():
 
 @pytest.fixture(scope="session")
 def default_seq2seq_lm():
-    return Seq2SeqLM("bigscience/mt0-small")
+    return Seq2SeqLM.fallback("bigscience/mt0-small")
 
 
 @pytest.fixture
@@ -28,6 +28,7 @@ def default_pb_request(default_pb_parameters, default_pb_stop_parameters):
     return generate_pb2.Request(
         id=0,
         inputs="Test",
+        input_chunks=generate_pb2.Input(chunks=[generate_pb2.InputChunk(text="Test")]),
         prefill_logprobs=True,
         truncate=100,
         parameters=default_pb_parameters,
@@ -61,7 +62,6 @@ def default_multi_requests_seq2seq_lm_batch(default_pb_request, mt0_small_tokeni
     )
 
 
-@pytest.mark.skip("seq2seq model not enabled on HPU yet")
 def test_batch_from_pb(default_pb_batch, default_seq2seq_lm_batch):
     batch = default_seq2seq_lm_batch
     sequence_length = len(default_seq2seq_lm_batch.input_ids[0])
@@ -93,18 +93,15 @@ def test_batch_from_pb(default_pb_batch, default_seq2seq_lm_batch):
     assert batch.max_decoder_input_length == batch.decoder_input_lengths[0]
 
 
-@pytest.mark.skip("seq2seq model not enabled on HPU yet")
 def test_batch_concatenate_no_prefill(default_seq2seq_lm_batch):
     with pytest.raises(ValueError):
         Seq2SeqLMBatch.concatenate([default_seq2seq_lm_batch, default_seq2seq_lm_batch])
 
 
-@pytest.mark.skip("seq2seq model not enabled on HPU yet")
 def test_seq2seq_lm_batch_type(default_seq2seq_lm):
     assert default_seq2seq_lm.batch_type == Seq2SeqLMBatch
 
 
-@pytest.mark.skip("seq2seq model not enabled on HPU yet")
 def test_seq2seq_lm_generate_token(default_seq2seq_lm, default_seq2seq_lm_batch):
     sequence_length = len(default_seq2seq_lm_batch.input_ids[0])
     generations, next_batch, _ = default_seq2seq_lm.generate_token(
@@ -172,7 +169,6 @@ def test_seq2seq_lm_generate_token(default_seq2seq_lm, default_seq2seq_lm_batch)
     assert generations[0].request_id == 0
 
 
-@pytest.mark.skip("seq2seq model not enabled on HPU yet")
 def test_seq2seq_lm_generate_token_completion(
     default_seq2seq_lm, default_seq2seq_lm_batch
 ):
@@ -190,7 +186,6 @@ def test_seq2seq_lm_generate_token_completion(
     assert generations[0].generated_text.generated_tokens == 7
 
 
-@pytest.mark.skip("seq2seq model not enabled on HPU yet")
 def test_seq2seq_lm_generate_token_completion_multi(
     default_seq2seq_lm, default_multi_requests_seq2seq_lm_batch
 ):
@@ -228,7 +223,6 @@ def test_seq2seq_lm_generate_token_completion_multi(
     assert generations[0].generated_text.generated_tokens == 7
 
 
-@pytest.mark.skip("seq2seq model not enabled on HPU yet")
 def test_batch_concatenate(
     default_seq2seq_lm,
     default_seq2seq_lm_batch,
@@ -287,7 +281,7 @@ def test_batch_concatenate(
     assert next_batch.max_decoder_input_length == 3
 
     assert next_batch.requests[0] == next_batch_0.requests[0]
-    assert next_batch.requests[1:] == next_batch_1.requests
+    assert next_batch.requests[1:] == list(next_batch_1.requests)
 
     assert next_batch.next_token_choosers[0] == next_batch_0.next_token_choosers[0]
     assert next_batch.next_token_choosers[1:] == next_batch_1.next_token_choosers
diff --git a/server/tests/models/test_starcoder.py b/server/tests/models/test_starcoder.py
deleted file mode 100644
index 05bdd72b..00000000
--- a/server/tests/models/test_starcoder.py
+++ /dev/null
@@ -1,372 +0,0 @@
-# Copyright (C) 2024 Habana Labs, Ltd. an Intel Company.
-
-import pytest
-import torch
-
-from copy import copy
-
-from text_generation_server.pb import generate_pb2
-from text_generation_server.models import get_model
-from text_generation_server.models.starcoder import StarCoderCausalLMBatch
-from text_generation_server.models.causal_lm import (
-    PREFILL_BATCH_BUCKET_SIZE,
-    PAD_SEQUENCE_TO_MULTIPLE_OF,
-    MAX_TOTAL_TOKENS,
-    BATCH_BUCKET_SIZE,
-)
-PAD_TOKEN=0
-
-
-@pytest.fixture(scope="session")
-def default_starcoder():
-    return get_model("bigcode/starcoder", None, None, None, None)
-
-
-@pytest.fixture(scope="session")
-def default_tokenizer(default_starcoder):
-    default_starcoder.tokenizer.pad_token_id = PAD_TOKEN
-    return default_starcoder.tokenizer
-
-
-@pytest.fixture
-def default_pb_request(default_pb_parameters, default_pb_stop_parameters):
-    return generate_pb2.Request(
-        id=0,
-        inputs="Test",
-        prefill_logprobs=True,
-        truncate=PAD_SEQUENCE_TO_MULTIPLE_OF,
-        parameters=default_pb_parameters,
-        stopping_parameters=default_pb_stop_parameters,
-    )
-
-
-@pytest.fixture
-def default_pb_batch(default_pb_request):
-    return generate_pb2.Batch(id=0, requests=[default_pb_request], size=1)
-
-
-@pytest.fixture
-def default_starcoder_batch(default_pb_batch, default_tokenizer):
-    return StarCoderCausalLMBatch.from_pb(
-        default_pb_batch, default_tokenizer, torch.float32, torch.device("hpu")
-    )
-
-
-@pytest.fixture
-def default_multi_requests_starcoder_batch(default_pb_request, default_tokenizer):
-    req_0 = copy(default_pb_request)
-    req_0.id = 1
-    req_1 = default_pb_request
-    req_1.id = 2
-    req_1.stopping_parameters.max_new_tokens = 5
-
-    batch_pb = generate_pb2.Batch(id=1, requests=[req_0, req_1], size=2)
-    return StarCoderCausalLMBatch.from_pb(
-        batch_pb, default_tokenizer, torch.float32, torch.device("hpu")
-    )
-
-
-def test_starcoder_batch_type(default_starcoder):
-    assert default_starcoder.batch_type == StarCoderCausalLMBatch
-
-
-def test_batch_from_pb(default_pb_batch, default_starcoder_batch):
-    batch = default_starcoder_batch
-
-    assert batch.batch_id == default_pb_batch.id
-    assert len(batch.requests) == len(default_pb_batch.requests)
-
-    for r in range(0,len(default_pb_batch.requests)):
-        assert batch.requests[r].data == default_pb_batch.requests[r]
-
-    # For Gaudi we are adding padding of multiplication of bucket size
-    size_of_padded_to_bucket = ((default_pb_batch.size + PREFILL_BATCH_BUCKET_SIZE - 1) // PREFILL_BATCH_BUCKET_SIZE) * PREFILL_BATCH_BUCKET_SIZE
-
-    assert len(batch.input_ids) == size_of_padded_to_bucket
-    assert batch.input_ids.shape == torch.Size([4, 128])
-
-    assert batch.input_ids[0][-2] == 1006
-    assert batch.input_ids[1][-2] == 49
-    assert batch.input_ids[2][-2] == 49
-    assert batch.attention_mask[0][-2] == 1
-    assert batch.attention_mask[1][-2] == 1
-    assert batch.attention_mask[2][-2] == 1
-    assert torch.all(batch.attention_mask[0, :-3] == 0)
-
-    assert batch.past_key_values is None
-    assert all(
-        [
-            torch.equal(input_ids, request.all_input_ids[:batch.input_length + 1, 0])
-            for input_ids, request in zip(batch.input_ids, batch.requests)
-        ]
-    )
-
-    assert len(batch) == default_pb_batch.size
-
-    assert batch.max_input_length + 1 == default_pb_batch.requests[0].truncate
-
-
-def test_starcoder_generate_token(default_starcoder, default_starcoder_batch):
-
-    sequence_length = len(default_starcoder_batch.requests[0].all_input_ids)
-    generations, next_batch, _ = default_starcoder.generate_token([default_starcoder_batch])
-    padding = next_batch.requests[0].stopping_criteria.max_new_tokens
-
-    assert isinstance(next_batch, StarCoderCausalLMBatch)
-    assert len(next_batch.attention_mask[0]) == PAD_SEQUENCE_TO_MULTIPLE_OF
-    assert next_batch.requests[0].all_input_ids[-padding-2] == 1006
-
-    assert torch.all(next_batch.requests[0].all_input_ids[-padding-1:] == PAD_TOKEN)
-    assert torch.all(next_batch.requests[0].all_input_ids[:-padding-3] == PAD_TOKEN)
-
-    generations, next_batch, _ = default_starcoder.generate_token([default_starcoder_batch])
-    assert torch.all(next_batch.attention_mask[0][PAD_SEQUENCE_TO_MULTIPLE_OF-2:PAD_SEQUENCE_TO_MULTIPLE_OF] == 1)
-    assert torch.all(next_batch.attention_mask[0][:PAD_SEQUENCE_TO_MULTIPLE_OF-3] == 0)
-    assert torch.all(next_batch.attention_mask[0][PAD_SEQUENCE_TO_MULTIPLE_OF+1:] == 0)
-
-    assert next_batch.requests[0].all_input_ids[-padding-2] == 1006
-    assert next_batch.requests[0].all_input_ids[-padding-1] == 26
-    assert torch.all(next_batch.requests[0].all_input_ids[-padding:] == PAD_TOKEN)
-    assert torch.all(next_batch.requests[0].all_input_ids[:-padding-3] == PAD_TOKEN)
-
-    assert next_batch.input_length == PAD_SEQUENCE_TO_MULTIPLE_OF
-    assert next_batch.max_input_length == next_batch.input_length
-
-    assert next_batch.past_key_values is not None
-    assert all(
-        [p[0].shape == (MAX_TOTAL_TOKENS, 256) for p in next_batch.past_key_values]
-    )
-    assert all(
-        [p[1].shape == (MAX_TOTAL_TOKENS, 256) for p in next_batch.past_key_values]
-    )
-    assert all([generation.generated_text is None for generation in generations])
-    assert all([len(generation.prefill_tokens) == PAD_SEQUENCE_TO_MULTIPLE_OF-1 for generation in generations])
-    assert all([generation.tokens.token_ids[0] == 26 for generation in generations])
-    assert all([generation.tokens.texts[0] == "(" for generation in generations])
-    assert generations[0].request_id == 0
-
-
-def test_starcoder_generate_token_completion(
-    default_starcoder, default_starcoder_batch
-):
-
-    next_batch = default_starcoder_batch
-    generations, next_batch, _ = default_starcoder.generate_token([next_batch])
-
-    for _ in range(default_starcoder_batch.requests[0].stopping_criteria.max_new_tokens - 1):
-        generations, next_batch, _ = default_starcoder.generate_token([next_batch])
-        assert len(generations) == len(next_batch)
-
-    generations, next_batch, _ = default_starcoder.generate_token([next_batch])
-
-    assert next_batch is None
-
-    assert len(generations) == 1
-    assert generations[0].generated_text.text == '(self):\n        """\n        Test that the test'
-    assert generations[0].request_id == default_starcoder_batch.requests[0].data.id
-    assert (
-        generations[0].generated_text.generated_tokens
-        == default_starcoder_batch.requests[0].stopping_criteria.max_new_tokens
-    )
-
-
-def test_starcoder_generate_token_completion_multi(
-    default_starcoder, default_multi_requests_starcoder_batch
-):
-    next_batch = default_multi_requests_starcoder_batch
-    generations, next_batch, _ = default_starcoder.generate_token([next_batch])
-
-    for i in range(
-        default_multi_requests_starcoder_batch.requests[1].stopping_criteria.max_new_tokens - 1
-    ):
-        generations, next_batch, _ = default_starcoder.generate_token([next_batch])
-        assert len(generations) == len(next_batch)
-
-    generations, next_batch, _ = default_starcoder.generate_token([next_batch])
-    assert next_batch is not None
-
-    assert len(generations) == 2
-    assert generations[1].generated_text.text == '(self):\n        """'
-    assert (
-        generations[1].request_id
-        == default_multi_requests_starcoder_batch.requests[1].data.id
-    )
-    assert (
-        generations[1].generated_text.generated_tokens
-        == default_multi_requests_starcoder_batch.requests[1].stopping_criteria.max_new_tokens
-    )
-
-    next_batch = next_batch.filter([next_batch.requests[0].data.id])
-
-    for _ in range(
-        default_multi_requests_starcoder_batch.requests[0].stopping_criteria.max_new_tokens - default_multi_requests_starcoder_batch.requests[1].stopping_criteria.max_new_tokens - 1
-    ):
-        generations, next_batch, _ = default_starcoder.generate_token([next_batch])
-        assert len(generations) == len(next_batch)
-
-    generations, next_batch, _ = default_starcoder.generate_token([next_batch])
-
-    assert next_batch is None
-
-    assert len(generations) == 1
-    assert generations[0].generated_text.text == '(self):\n        """\n        Test that the test'
-    assert (
-        generations[0].request_id
-        == default_multi_requests_starcoder_batch.requests[0].data.id
-    )
-    assert (
-        generations[0].generated_text.generated_tokens
-        == default_multi_requests_starcoder_batch.requests[0].stopping_criteria.max_new_tokens
-    )
-
-
-def test_batch_concatenate(
-    default_starcoder, default_starcoder_batch, default_multi_requests_starcoder_batch
-):
-    next_batch_0 = default_starcoder_batch
-    _, next_batch_0, _ = default_starcoder.generate_token([next_batch_0])
-    _, next_batch_0, _ = default_starcoder.generate_token([next_batch_0])
-    _, next_batch_0, _ = default_starcoder.generate_token([next_batch_0])
-
-    next_batch_1 = default_multi_requests_starcoder_batch
-    _, next_batch_1, _ = default_starcoder.generate_token([next_batch_1])
-    _, next_batch_1, _ = default_starcoder.generate_token([next_batch_1])
-
-    # Clone past_key_values before concatenating to compare after,
-    # because they are removed from the concatenated batches
-    next_batch_0_past_key_values = [x.clone() for x in next_batch_0.past_key_values]
-    next_batch_1_past_key_values = [x.clone() for x in next_batch_1.past_key_values]
-
-    next_batch = StarCoderCausalLMBatch.concatenate([next_batch_0, next_batch_1])
-
-    assert torch.equal(next_batch.requests[0].all_input_ids, next_batch_0.requests[0].all_input_ids)
-    assert torch.equal(next_batch.requests[1].all_input_ids, next_batch_1.requests[0].all_input_ids)
-    assert torch.equal(next_batch.requests[2].all_input_ids, next_batch_1.requests[1].all_input_ids)
-
-
-    assert torch.all(
-            next_batch.attention_mask[0:2, -next_batch.right_padding - 2: -next_batch.right_padding] == 1
-    )
-    assert torch.all(
-            next_batch.attention_mask[2, -next_batch.right_padding - 3: -next_batch.right_padding] == 1
-    )
-    assert torch.all(
-            next_batch.attention_mask[3, -next_batch.right_padding - 2: -next_batch.right_padding] == 1
-    )
-
-    assert torch.all(
-        next_batch.attention_mask[0:2, :-next_batch.right_padding-2] == 0)
-    assert torch.all(
-        next_batch.attention_mask[2, :-next_batch.right_padding-4] == 0)
-    assert torch.all(
-        next_batch.attention_mask[3, :-next_batch.right_padding-3] == 0)
-
-    assert next_batch.batch_id == 0
-    assert next_batch.input_ids[0,-next_batch.right_padding - 2] == 1006
-    assert next_batch.input_ids[0,-next_batch.right_padding - 1] == 26
-
-    assert next_batch.max_input_length == 129
-
-    assert torch.all(next_batch.input_ids[0,-next_batch.right_padding:] == PAD_TOKEN)
-    assert torch.all(next_batch.input_ids[1,-next_batch.right_padding:] == PAD_TOKEN)
-    assert torch.all(next_batch.input_ids[2,-next_batch.right_padding:] == PAD_TOKEN)
-    assert torch.all(next_batch.input_ids[3,-next_batch.right_padding:] == PAD_TOKEN)
-
-    assert next_batch.input_length == PAD_SEQUENCE_TO_MULTIPLE_OF  +1
-    assert next_batch.max_input_length == PAD_SEQUENCE_TO_MULTIPLE_OF + 1
-
-    assert next_batch.requests[0] == next_batch_0.requests[0]
-    assert next_batch.requests[1:] == next_batch_1.requests
-
-    assert next_batch.requests[0].stopping_criteria == next_batch_0.requests[0].stopping_criteria
-    assert next_batch.requests[1].stopping_criteria == next_batch_1.requests[0].stopping_criteria
-    assert next_batch.requests[2].stopping_criteria == next_batch_1.requests[1].stopping_criteria
-
-    assert next_batch.past_key_values is not None
-
-    assert all([p[0].shape == (2048, 256) for p in next_batch.past_key_values])
-    assert all([p[1].shape == (2048, 256) for p in next_batch.past_key_values])
-
-    assert next_batch.past_key_values is not None
-
-    for i, past in enumerate(next_batch.past_key_values):
-        assert torch.equal(next_batch_0_past_key_values[i][0,0,0:128], past[0][1:129][0, 0:128])
-        assert torch.equal(next_batch_0_past_key_values[i][0,1,0:128], past[1][1:129][0, 0:128])
-        assert torch.equal(
-            next_batch_1_past_key_values[i][:, :, 0:1][0][0][0], past[0][1:, :][0][0]
-        )
-
-        assert torch.equal(
-            next_batch_1_past_key_values[i][1:, :, 0:1][0][0][0], past[1][1:, :][0][0]
-        )
-
-    generations, next_batch, _ = default_starcoder.generate_token([next_batch])
-
-    for _ in range(
-        default_multi_requests_starcoder_batch.requests[1].stopping_criteria.max_new_tokens - 2
-    ):
-        generations, next_batch, _ = default_starcoder.generate_token([next_batch])
-        assert len(generations) == len(next_batch)
-
-    generations, next_batch, _ = default_starcoder.generate_token([next_batch])
-    assert next_batch is not None
-
-    assert len(generations) == 3
-    assert generations[2].generated_text.text == '(self):\n        """'
-
-    assert (
-        generations[2].request_id
-        == default_multi_requests_starcoder_batch.requests[1].data.id
-    )
-    assert (
-        generations[2].generated_text.generated_tokens
-        == default_multi_requests_starcoder_batch.requests[1].stopping_criteria.max_new_tokens
-    )
-
-    next_batch = next_batch.filter(
-        [next_batch.requests[0].data.id, next_batch.requests[1].data.id]
-    )
-
-    for _ in range(
-        default_starcoder_batch.requests[0].stopping_criteria.max_new_tokens
-        - default_multi_requests_starcoder_batch.requests[1].stopping_criteria.max_new_tokens
-        - 2
-    ):
-        generations, next_batch, _ = default_starcoder.generate_token([next_batch])
-        assert len(generations) == len(next_batch)
-
-    generations, next_batch, _ = default_starcoder.generate_token([next_batch])
-    assert next_batch is not None
-
-    assert len(generations) == 2
-    assert generations[0].generated_text.text == '(self):\n        """\n        Test that the test'
-    assert generations[0].request_id == default_starcoder_batch.requests[0].data.id
-    assert (
-        generations[0].generated_text.generated_tokens
-        == default_starcoder_batch.requests[0].stopping_criteria.max_new_tokens
-    )
-
-    next_batch = next_batch.filter([next_batch.requests[1].data.id])
-
-    for _ in range(
-        default_multi_requests_starcoder_batch.requests[0].stopping_criteria.max_new_tokens
-        - default_starcoder_batch.requests[0].stopping_criteria.max_new_tokens
-        - default_multi_requests_starcoder_batch.requests[1].stopping_criteria.max_new_tokens
-        - 4
-    ):
-        generations, next_batch, _ = default_starcoder.generate_token([next_batch])
-        assert len(generations) == len(next_batch)
-
-    generations, next_batch, _ = default_starcoder.generate_token([next_batch])
-    assert next_batch is None
-
-    assert len(generations) == 1
-    assert generations[0].generated_text.text == '(self):\n        """\n        Test that the test'
-    assert (
-        generations[0].request_id
-        == default_multi_requests_starcoder_batch.requests[0].data.id
-    )
-    assert (
-        generations[0].generated_text.generated_tokens
-        == default_multi_requests_starcoder_batch.requests[0].stopping_criteria.max_new_tokens
-    )
diff --git a/server/tests/utils/test_adapter.py b/server/tests/utils/test_adapter.py
new file mode 100644
index 00000000..a27c1055
--- /dev/null
+++ b/server/tests/utils/test_adapter.py
@@ -0,0 +1,247 @@
+import pytest
+from unittest.mock import Mock
+from text_generation_server.utils.adapter import (
+    get_attn_weights,
+    get_mlp_weights,
+    parse_lora_adapters,
+    AdapterInfo,
+)
+
+
+def test_parse_lora_adapters_empty():
+    assert parse_lora_adapters(None) == []
+    assert parse_lora_adapters("") == []
+
+
+def test_parse_lora_adapters_single():
+    result = parse_lora_adapters("adapter1")
+    assert result == [AdapterInfo(id="adapter1", path=None, revision=None)]
+
+
+def test_parse_lora_adapters_with_path():
+    result = parse_lora_adapters("adapter1=path/to/adapter1")
+    assert result == [
+        AdapterInfo(id="adapter1", path="path/to/adapter1", revision=None)
+    ]
+
+
+def test_parse_lora_adapters_with_path_and_revision():
+    result = parse_lora_adapters("adapter1=path/to/adapter1@main")
+    assert result == [
+        AdapterInfo(id="adapter1", path="path/to/adapter1", revision="main")
+    ]
+
+
+def test_parse_lora_adapters_multiple():
+    result = parse_lora_adapters(
+        "adapter1,adapter2=path/to/adapter2,adapter3=path/to/adapter3@dev"
+    )
+    assert result == [
+        AdapterInfo(id="adapter1", path=None, revision=None),
+        AdapterInfo(id="adapter2", path="path/to/adapter2", revision=None),
+        AdapterInfo(id="adapter3", path="path/to/adapter3", revision="dev"),
+    ]
+
+
+def test_parse_lora_adapters_invalid_format():
+    try:
+        parse_lora_adapters("adapter1,invalid=format=test,adapter3")
+        assert False, "Should have raised ValueError"
+    except ValueError as e:
+        assert str(e) == "Invalid LoRA adapter format: invalid=format=test"
+
+
+def test_get_attn_weights():
+    # create a mock layer
+    mock_layer = Mock()
+    mock_layer.self_attn.query_key_value = Mock()
+    mock_layer.self_attn.o_proj = Mock()
+
+    # call the function
+    result = get_attn_weights(2, mock_layer)
+
+    # assert the result
+    expected = {
+        (2, "q_proj"): (
+            "model.layers.2.self_attn.q_proj",
+            mock_layer.self_attn.query_key_value,
+        ),
+        (2, "k_proj"): (
+            "model.layers.2.self_attn.k_proj",
+            mock_layer.self_attn.query_key_value,
+        ),
+        (2, "qkv_proj"): (
+            "model.layers.2.self_attn.qkv_proj",
+            mock_layer.self_attn.query_key_value,
+        ),
+        (2, "v_proj"): (
+            "model.layers.2.self_attn.v_proj",
+            mock_layer.self_attn.query_key_value,
+        ),
+        (2, "o_proj"): ("model.layers.2.self_attn.o_proj", mock_layer.self_attn.o_proj),
+    }
+    assert result == expected
+
+
+def test_get_mlp_weights_with_gate_up_proj():
+    # create a mock layer with gate_up_proj
+    mock_layer = Mock()
+    mock_layer.mlp.gate_up_proj = Mock()
+    mock_layer.mlp.down_proj = Mock()
+
+    # call the function
+    result = get_mlp_weights(3, mock_layer)
+
+    # assert the result
+    expected = {
+        (3, "gate_proj"): ("model.layers.3.mlp.gate_proj", mock_layer.mlp.gate_up_proj),
+        (3, "up_proj"): ("model.layers.3.mlp.up_proj", mock_layer.mlp.gate_up_proj),
+        (3, "down_proj"): ("model.layers.3.mlp.down_proj", mock_layer.mlp.down_proj),
+    }
+    assert result == expected
+
+
+def test_get_mlp_weights_without_gate_up_proj():
+    # create a mock layer without gate_up_proj
+    mock_layer = Mock()
+    mock_layer.mlp = Mock(spec=[])
+
+    # call the function
+    result = get_mlp_weights(1, mock_layer)
+
+    # assert the result
+    assert result == {}
+
+
+@pytest.mark.parametrize("layer_index", [0, 1, 5])
+def test_get_attn_weights_different_layers(layer_index):
+    mock_layer = Mock()
+    mock_layer.self_attn.query_key_value = Mock()
+    mock_layer.self_attn.o_proj = Mock()
+
+    result = get_attn_weights(layer_index, mock_layer)
+
+    for k in ["q", "k", "v"]:
+        assert (layer_index, f"{k}_proj") in result
+        assert (
+            result[(layer_index, f"{k}_proj")][0]
+            == f"model.layers.{layer_index}.self_attn.{k}_proj"
+        )
+
+    assert (layer_index, "o_proj") in result
+    assert (
+        result[(layer_index, "o_proj")][0]
+        == f"model.layers.{layer_index}.self_attn.o_proj"
+    )
+
+
+@pytest.mark.parametrize("layer_index", [0, 1, 5])
+def test_get_mlp_weights_different_layers(layer_index):
+    mock_layer = Mock()
+    mock_layer.mlp.gate_up_proj = Mock()
+    mock_layer.mlp.down_proj = Mock()
+
+    result = get_mlp_weights(layer_index, mock_layer)
+
+    for k in ["gate", "up", "down"]:
+        assert (layer_index, f"{k}_proj") in result
+        assert (
+            result[(layer_index, f"{k}_proj")][0]
+            == f"model.layers.{layer_index}.mlp.{k}_proj"
+        )
+
+
+def test_get_attn_weights_llama_compatibility():
+    mock_layer = Mock()
+    mock_layer.self_attn.query_key_value = Mock()
+    mock_layer.self_attn.o_proj = Mock()
+
+    result = get_attn_weights(2, mock_layer)
+
+    expected = {
+        (2, "q_proj"): (
+            "model.layers.2.self_attn.q_proj",
+            mock_layer.self_attn.query_key_value,
+        ),
+        (2, "k_proj"): (
+            "model.layers.2.self_attn.k_proj",
+            mock_layer.self_attn.query_key_value,
+        ),
+        (2, "qkv_proj"): (
+            "model.layers.2.self_attn.qkv_proj",
+            mock_layer.self_attn.query_key_value,
+        ),
+        (2, "v_proj"): (
+            "model.layers.2.self_attn.v_proj",
+            mock_layer.self_attn.query_key_value,
+        ),
+        (2, "o_proj"): ("model.layers.2.self_attn.o_proj", mock_layer.self_attn.o_proj),
+    }
+    assert result == expected
+
+
+def test_get_mlp_weights_llama_compatibility():
+    mock_layer = Mock()
+    mock_layer.mlp.gate_up_proj = Mock()
+    mock_layer.mlp.down_proj = Mock()
+
+    result = get_mlp_weights(3, mock_layer)
+
+    expected = {
+        (3, "gate_proj"): ("model.layers.3.mlp.gate_proj", mock_layer.mlp.gate_up_proj),
+        (3, "up_proj"): ("model.layers.3.mlp.up_proj", mock_layer.mlp.gate_up_proj),
+        (3, "down_proj"): ("model.layers.3.mlp.down_proj", mock_layer.mlp.down_proj),
+    }
+    assert result == expected
+
+
+def test_get_attn_weights_gemma_compatibility():
+    mock_layer = Mock()
+    mock_layer.self_attn.query_key_value = Mock()
+    mock_layer.self_attn.o_proj = Mock()
+
+    result = get_attn_weights(2, mock_layer)
+
+    expected = {
+        (2, "q_proj"): (
+            "model.layers.2.self_attn.q_proj",
+            mock_layer.self_attn.query_key_value,
+        ),
+        (2, "k_proj"): (
+            "model.layers.2.self_attn.k_proj",
+            mock_layer.self_attn.query_key_value,
+        ),
+        (2, "qkv_proj"): (
+            "model.layers.2.self_attn.qkv_proj",
+            mock_layer.self_attn.query_key_value,
+        ),
+        (2, "v_proj"): (
+            "model.layers.2.self_attn.v_proj",
+            mock_layer.self_attn.query_key_value,
+        ),
+        (2, "o_proj"): ("model.layers.2.self_attn.o_proj", mock_layer.self_attn.o_proj),
+    }
+    assert result == expected
+
+
+def test_get_mlp_weights_gemma_compatibility():
+    mock_layer = Mock()
+    mock_layer.mlp.gate_proj = Mock()
+    mock_layer.mlp.up_proj = Mock()
+    mock_layer.mlp.down_proj = Mock()
+
+    # ensure that the mock_layer.mlp.gate_up_proj attribute does not exist.
+    # This is necessary because the use of `Mock` automatically creates any
+    # attributes that are accessed, even if they don't exist in the actual
+    # implementation. If `gate_up_proj` were created, `get_mlp_weights` might
+    # follow the wrong execution path and return an incorrect result.
+    del mock_layer.mlp.gate_up_proj
+
+    result = get_mlp_weights(3, mock_layer)
+
+    expected = {
+        (3, "gate_proj"): ("model.layers.3.mlp.gate_proj", mock_layer.mlp.gate_proj),
+        (3, "up_proj"): ("model.layers.3.mlp.up_proj", mock_layer.mlp.up_proj),
+        (3, "down_proj"): ("model.layers.3.mlp.down_proj", mock_layer.mlp.down_proj),
+    }
+    assert result == expected
diff --git a/server/tests/utils/test_hub.py b/server/tests/utils/test_hub.py
index 721820f5..291a41b0 100644
--- a/server/tests/utils/test_hub.py
+++ b/server/tests/utils/test_hub.py
@@ -1,11 +1,9 @@
 import os
-import requests
 import tempfile
 
 import pytest
 
 import huggingface_hub.constants
-from huggingface_hub import hf_api
 
 import text_generation_server.utils.hub
 from text_generation_server.utils.hub import (
diff --git a/server/tests/utils/test_layers.py b/server/tests/utils/test_layers.py
index 9a8da0d6..118540ee 100644
--- a/server/tests/utils/test_layers.py
+++ b/server/tests/utils/test_layers.py
@@ -42,7 +42,12 @@ class Weights:
 def test_weight_hub_files_offline_error():
 
     vocab_size = 17
-    weights = Weights(rank=0, world_size=1, vocab_size=vocab_size, hidden_dim=256)
+    weights = Weights(
+        rank=0,
+        world_size=1,
+        vocab_size=vocab_size,
+        hidden_dim=256,
+    )
     embeddings = TensorParallelEmbedding("", weights)
 
     input_ids = torch.arange(vocab_size)
diff --git a/server/tests/utils/test_tokens.py b/server/tests/utils/test_tokens.py
index 94d2a8f2..5db32776 100644
--- a/server/tests/utils/test_tokens.py
+++ b/server/tests/utils/test_tokens.py
@@ -1,26 +1,12 @@
-# Copyright (C) 2024 Habana Labs, Ltd. an Intel Company.
-
-import pytest
 import torch
-from transformers import AutoTokenizer
-
 from text_generation_server.utils.tokens import (
     StopSequenceCriteria,
     StoppingCriteria,
     FinishReason,
     batch_top_tokens,
-    make_tokenizer_optional,
 )
 
 
-@pytest.fixture
-def skip_tokenizer_env_var():
-    import os
-    os.environ["SKIP_TOKENIZER_IN_TGI"] = "true"
-    yield
-    del os.environ['SKIP_TOKENIZER_IN_TGI']
-
-
 def test_stop_sequence_criteria():
     criteria = StopSequenceCriteria("/test;")
 
@@ -100,33 +86,3 @@ def test_batch_top_tokens():
     assert topn_tok_logprobs[2] == [[-1, -2, -3, -3]]
     assert topn_tok_logprobs[3] == [[-1, -2, -3, -3]]
     assert topn_tok_logprobs[4] == [[-1, -2, -3, -3, -4]]
-
-
-def test_pass_through_tokenizer(skip_tokenizer_env_var):
-    tokenizer = AutoTokenizer.from_pretrained(
-        'meta-llama/Llama-2-7b-chat-hf',
-        revision=None,
-        padding_side="left",
-        truncation_side="left",
-    )
-    tokenizer.pad_token_id = 2
-    make_tokenizer_optional(tokenizer)
-
-    input = ["1,  1724, 338,  6483,  6509, 29973", "?"]
-    tokenized_inputs = tokenizer(
-        input,
-        return_tensors="pt",
-        padding="max_length",
-        return_token_type_ids=False,
-        truncation=True,
-        max_length=1024,
-    )
-    assert tokenized_inputs['input_ids'].size() == torch.Size([2, 1024])
-    assert torch.equal(tokenized_inputs['input_ids'][0][1018:], torch.tensor([1, 1724, 338, 6483, 6509, 29973]))
-    assert torch.equal(tokenized_inputs['input_ids'][1][1023:], torch.tensor([tokenizer.pad_token_id]))
-    decoded_tokens = tokenizer.decode(tokenized_inputs["input_ids"][0], skip_special_tokens=True, clean_up_tokenization_spaces=False)
-    assert decoded_tokens.split(',')[1018:] == ['1', '1724', '338', '6483', '6509', '29973']
-
-
-if __name__ == "__main__":
-    test_pass_through_tokenizer()
diff --git a/server/tests/utils/test_watermark.py b/server/tests/utils/test_watermark.py
index c7c83e8c..5d0aaf05 100644
--- a/server/tests/utils/test_watermark.py
+++ b/server/tests/utils/test_watermark.py
@@ -1,8 +1,7 @@
-# Copyright (C) 2024 Habana Labs, Ltd. an Intel Company.
+# test_watermark_logits_processor.py
 
 import os
 import numpy as np
-import pytest
 import torch
 from text_generation_server.utils.watermark import WatermarkLogitsProcessor
 
@@ -11,71 +10,35 @@ GAMMA = os.getenv("WATERMARK_GAMMA", 0.5)
 DELTA = os.getenv("WATERMARK_DELTA", 2.0)
 
 
-@pytest.fixture
-def hpu_device():
-    return torch.device("hpu")
-
-
-@pytest.fixture
-def input_ids_list():
-    return [101, 2036, 3731, 102, 2003, 103]
-
-
-@pytest.fixture
-def input_ids_tensor(hpu_device):
-    return torch.tensor(
-        [[101, 2036, 3731, 102, 2003, 103]],
-        dtype=torch.int64,
-        device=hpu_device
-    )
-
-
-@pytest.fixture
-def scores(hpu_device):
-    return torch.tensor(
-        [[0.5, 0.3, 0.2, 0.8], [0.1, 0.2, 0.7, 0.9]],
-        device=hpu_device
-    )
-
-
-def test_seed_rng(input_ids_list, hpu_device):
-    processor = WatermarkLogitsProcessor(device=hpu_device)
-    processor._seed_rng(input_ids_list)
+def test_seed_rng():
+    input_ids = [101, 2036, 3731, 102, 2003, 103]
+    processor = WatermarkLogitsProcessor()
+    processor._seed_rng(input_ids)
     assert isinstance(processor.rng, torch.Generator)
 
 
-def test_seed_rng_tensor(input_ids_tensor, hpu_device):
-    processor = WatermarkLogitsProcessor(device=hpu_device)
-    processor._seed_rng(input_ids_tensor)
-    assert isinstance(processor.rng, torch.Generator)
-
-
-def test_get_greenlist_ids(input_ids_list, hpu_device):
-    processor = WatermarkLogitsProcessor(device=hpu_device)
-    result = processor._get_greenlist_ids(input_ids_list, 10, hpu_device)
+def test_get_greenlist_ids():
+    input_ids = [101, 2036, 3731, 102, 2003, 103]
+    processor = WatermarkLogitsProcessor()
+    result = processor._get_greenlist_ids(input_ids, 10, torch.device("cpu"))
     assert max(result) <= 10
     assert len(result) == int(10 * 0.5)
 
 
-def test_get_greenlist_ids_tensor(input_ids_tensor, hpu_device):
-    processor = WatermarkLogitsProcessor(device=hpu_device)
-    result = processor._get_greenlist_ids(input_ids_tensor, 10, hpu_device)
-    assert max(result) <= 10
-    assert len(result) == int(10 * 0.5)
-
-
-def test_calc_greenlist_mask(scores, hpu_device):
-    processor = WatermarkLogitsProcessor(device=hpu_device)
-    greenlist_token_ids = torch.tensor([2, 3], device=hpu_device)
+def test_calc_greenlist_mask():
+    processor = WatermarkLogitsProcessor()
+    scores = torch.tensor([[0.5, 0.3, 0.2, 0.8], [0.1, 0.2, 0.7, 0.9]])
+    greenlist_token_ids = torch.tensor([2, 3])
     result = processor._calc_greenlist_mask(scores, greenlist_token_ids)
     assert result.tolist() == [[False, False, False, False], [False, False, True, True]]
     assert result.shape == scores.shape
 
 
-def test_bias_greenlist_logits(scores, hpu_device):
-    processor = WatermarkLogitsProcessor(device=hpu_device)
+def test_bias_greenlist_logits():
+    processor = WatermarkLogitsProcessor()
+    scores = torch.tensor([[0.5, 0.3, 0.2, 0.8], [0.1, 0.2, 0.7, 0.9]])
     green_tokens_mask = torch.tensor(
-        [[False, False, True, True], [False, False, False, True]], device=hpu_device
+        [[False, False, True, True], [False, False, False, True]]
     )
     greenlist_bias = 2.0
     result = processor._bias_greenlist_logits(scores, green_tokens_mask, greenlist_bias)
@@ -83,13 +46,9 @@ def test_bias_greenlist_logits(scores, hpu_device):
     assert result.shape == scores.shape
 
 
-def test_call(input_ids_list, scores, hpu_device):
-    processor = WatermarkLogitsProcessor(device=hpu_device)
-    result = processor(input_ids_list, scores)
-    assert result.shape == scores.shape
-
-
-def test_call_tensor(input_ids_tensor, scores, hpu_device):
-    processor = WatermarkLogitsProcessor(device=hpu_device)
-    result = processor(input_ids_tensor, scores)
+def test_call():
+    input_ids = [101, 2036, 3731, 102, 2003, 103]
+    processor = WatermarkLogitsProcessor()
+    scores = torch.tensor([[0.5, 0.3, 0.2, 0.8], [0.1, 0.2, 0.7, 0.9]])
+    result = processor(input_ids, scores)
     assert result.shape == scores.shape
diff --git a/server/tests/utils/test_weights.py b/server/tests/utils/test_weights.py
new file mode 100644
index 00000000..556fcea1
--- /dev/null
+++ b/server/tests/utils/test_weights.py
@@ -0,0 +1,1177 @@
+import pytest
+import torch
+from text_generation_server.utils.weights import (
+    DefaultWeightsLoader,
+    Weights,
+    WeightsLoader,
+)
+from text_generation_server.layers.gptq import GPTQWeight, GPTQWeightsLoader
+from text_generation_server.layers.exl2 import Exl2Weight, Exl2WeightsLoader
+from text_generation_server.layers.marlin.marlin import (
+    MarlinWeight,
+    MarlinWeightsLoader,
+)
+from types import SimpleNamespace
+from typing import List, Optional, Dict, Union
+from pathlib import Path
+
+
+@pytest.fixture
+def gptq_weights_loader():
+    return GPTQWeightsLoader(
+        bits=4,
+        groupsize=-1,
+        desc_act=False,
+        quant_method="gptq",
+        quantize="gptq",
+        sym=True,
+    )
+
+
+@pytest.fixture
+def gptq_weights_loader_awq():
+    return GPTQWeightsLoader(
+        bits=4,
+        groupsize=-1,
+        desc_act=False,
+        quant_method="awq",
+        quantize="awq",
+        sym=True,
+    )
+
+
+@pytest.fixture
+def marlin_weights_loader():
+    return MarlinWeightsLoader(bits=4, is_marlin_24=False)
+
+
+dummy_file_system = {
+    "test_weights": {
+        "layer.0.weight": torch.tensor(
+            [
+                [1, 2],
+                [3, 4],
+            ],
+            dtype=torch.float32,
+        ),
+    },
+    "test_weights_2": {
+        "layer.1337.weight": torch.tensor(
+            [
+                [1, 2, 3, 4],
+                [5, 6, 7, 8],
+            ],
+            dtype=torch.float32,
+        ),
+    },
+    "test_get_weights_col_packed": {
+        "weight.weight": torch.tensor(
+            [
+                [1, 2],
+                [3, 4],
+                [5, 6],
+                [7, 8],
+            ],
+            dtype=torch.float32,
+        ),
+    },
+    "test_get_multi_weights_col": {
+        "weight.weight": torch.tensor(
+            [
+                [1, 2],
+                [3, 4],
+                [5, 6],
+                [7, 8],
+            ],
+            dtype=torch.float32,
+        ),
+    },
+    "test_get_weights_row": {
+        "weight.weight": torch.tensor(
+            [
+                [1, 2],
+                [3, 4],
+                [5, 6],
+                [7, 8],
+            ],
+            dtype=torch.float32,
+        ),
+    },
+    "test_get_weights_col_gptq": {
+        "weight.qweight": torch.tensor(
+            [
+                [1, 2],
+                [3, 4],
+                [5, 6],
+                [7, 8],
+            ],
+            dtype=torch.float32,
+        ),
+        "weight.g_idx": torch.tensor([0, 1, 0, 1], dtype=torch.int32),
+        "weight.qzeros": torch.tensor(
+            [
+                [0, 1],
+                [1, 0],
+            ],
+            dtype=torch.int32,
+        ),
+        "weight.scales": torch.tensor(
+            [
+                [100.0, 100.0],
+                [100.0, 100.0],
+            ],
+            dtype=torch.float16,
+        ),
+        "gptq_bits": torch.tensor([8], dtype=torch.float32),
+        "gptq_groupsize": torch.tensor([2], dtype=torch.float32),
+    },
+    "test_get_weights_col_marlin": {
+        "weight.B": torch.tensor([[1, 2], [3, 4]], dtype=torch.int32),
+        "weight.s": torch.tensor([[0.5000], [0.2500]], dtype=torch.float16),
+    },
+    "test_get_weights_row_gptq": {
+        "weight.qweight": torch.tensor(
+            [
+                [1, 2],
+                [3, 4],
+                [5, 6],
+                [7, 8],
+            ],
+            dtype=torch.int32,
+        ),
+        "weight.g_idx": torch.tensor([0, 1, 0, 1], dtype=torch.int32),
+        "weight.qzeros": torch.tensor(
+            [
+                [0, 1],
+                [1, 0],
+            ],
+            dtype=torch.int32,
+        ),
+        "weight.scales": torch.tensor(
+            [
+                [100.0, 100.0],
+                [100.0, 100.0],
+            ],
+            dtype=torch.float16,
+        ),
+        "gptq_bits": torch.tensor([8], dtype=torch.float32),
+        "gptq_groupsize": torch.tensor([2], dtype=torch.float32),
+    },
+    "test_get_multi_weights_col_gptq": {
+        "weight.qweight": torch.tensor(
+            [
+                [1, 2],
+                [3, 4],
+                [5, 6],
+                [7, 8],
+            ],
+            dtype=torch.int32,
+        ),
+        "weight.g_idx": torch.tensor([0, 1, 0, 1], dtype=torch.int32),
+        "weight.qzeros": torch.tensor(
+            [
+                [0, 1],
+                [1, 0],
+            ],
+            dtype=torch.int32,
+        ),
+        "weight.scales": torch.tensor(
+            [
+                [100.0, 100.0],
+                [100.0, 100.0],
+            ],
+            dtype=torch.float16,
+        ),
+        "gptq_bits": torch.tensor([8], dtype=torch.float32),
+        "gptq_groupsize": torch.tensor([2], dtype=torch.float32),
+    },
+    "test_get_weights_col_packed_gptq": {
+        "weight.qweight": torch.tensor(
+            [
+                [1, 2],
+                [3, 4],
+                [5, 6],
+                [7, 8],
+            ],
+            dtype=torch.int32,
+        ),
+        "weight.g_idx": torch.tensor([0, 1, 0, 1], dtype=torch.int32),
+        "weight.qzeros": torch.tensor(
+            [
+                [0, 1],
+                [1, 0],
+            ],
+            dtype=torch.int32,
+        ),
+        "weight.scales": torch.tensor(
+            [
+                [100.0, 100.0],
+                [100.0, 100.0],
+            ],
+            dtype=torch.float16,
+        ),
+        "gptq_bits": torch.tensor([8], dtype=torch.float32),
+        "gptq_groupsize": torch.tensor([2], dtype=torch.float32),
+    },
+    "test_get_weights_col_packed_exl2": {
+        "weight.q_weight": torch.tensor(
+            [
+                [1, 2],
+                [3, 4],
+                [5, 6],
+                [7, 8],
+            ],
+            dtype=torch.int32,
+        ),
+        "weight.q_scale": torch.tensor([8], dtype=torch.int32),
+        "weight.q_invperm": torch.tensor([1, 0, 3, 2], dtype=torch.int32),
+        "weight.q_scale_max": torch.tensor([100], dtype=torch.float16),
+        "weight.q_groups": torch.tensor([4], dtype=torch.int16),
+    },
+    "test_get_weights_row_exl2": {
+        "weight.q_weight": torch.tensor(
+            [
+                [1, 2],
+                [3, 4],
+                [5, 6],
+                [7, 8],
+            ],
+            dtype=torch.int32,
+        ),
+        "weight.q_scale": torch.tensor([8], dtype=torch.int32),
+        "weight.q_invperm": torch.tensor([1, 0, 3, 2], dtype=torch.int32),
+        "weight.q_scale_max": torch.tensor([100], dtype=torch.float16),
+        "weight.q_groups": torch.tensor([4], dtype=torch.int16),
+    },
+    "test_get_multi_weights_col_exl2": {
+        "weight.q_weight": torch.tensor(
+            [
+                [1, 2],
+                [3, 4],
+                [5, 6],
+                [7, 8],
+            ],
+            dtype=torch.int32,
+        ),
+        "weight.q_scale": torch.tensor([8], dtype=torch.int32),
+        "weight.q_invperm": torch.tensor([1, 0, 3, 2], dtype=torch.int32),
+        "weight.q_scale_max": torch.tensor([100], dtype=torch.float16),
+        "weight.q_groups": torch.tensor([4], dtype=torch.int16),
+    },
+    "test_get_weights_col_exl2": {
+        "weight.q_weight": torch.tensor(
+            [
+                [1, 2],
+                [3, 4],
+                [5, 6],
+                [7, 8],
+            ],
+            dtype=torch.int32,
+        ),
+        "weight.q_scale": torch.tensor([8], dtype=torch.int32),
+        "weight.q_invperm": torch.tensor([1, 0, 3, 2], dtype=torch.int32),
+        "weight.q_scale_max": torch.tensor([100], dtype=torch.float16),
+        "weight.q_groups": torch.tensor([4], dtype=torch.int16),
+    },
+    "test_get_weights_row_marlin": {
+        "weight.B": torch.tensor([[1, 2], [3, 4]], dtype=torch.int32),
+        "weight.s": torch.tensor([[0.5], [0.25]], dtype=torch.float16),
+    },
+    "test_get_multi_weights_col_marlin": {
+        "weight.B": torch.tensor([[1, 2], [3, 4]], dtype=torch.int32),
+        "weight.s": torch.tensor([[0.5], [0.25]], dtype=torch.float16),
+    },
+    "test_get_weights_col_packed_marlin": {
+        "weight.B": torch.tensor([[1, 2], [3, 4]], dtype=torch.int32),
+        "weight.s": torch.tensor([[0.5], [0.25]], dtype=torch.float16),
+    },
+}
+
+
+class MockSlice:
+    def __init__(self, tensor):
+        self.tensor = tensor
+
+    def get_shape(self):
+        return self.tensor.shape
+
+    def __getitem__(self, idx):
+        return self.tensor[idx]
+
+
+def mock_get_slice(tensor_name, filename):
+    tensor = dummy_file_system[filename][tensor_name]
+    return MockSlice(tensor)
+
+
+def mock_handle(filename, device, dtype):
+    return SimpleNamespace(
+        get_slice=lambda tensor_name: mock_get_slice(tensor_name, filename)
+    )
+
+
+class MockSafeOpen:
+    def __init__(self, filename, framework, dummy_fs):
+        self.filename = filename
+        self.framework = framework
+        self.dummy_fs = dummy_fs
+
+    def keys(self):
+        return list(self.dummy_fs[self.filename].keys())
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        pass
+
+
+class MockWeights(Weights):
+    def __init__(
+        self,
+        filenames: List[Union[Path, str]],
+        device,
+        dtype,
+        process_group,
+        dummy_fs,
+        aliases: Optional[Dict[str, List[str]]] = None,
+        prefix: Optional[str] = None,
+        weights_loader: Optional[WeightsLoader] = None,
+    ):
+        routing = {}
+        self.dummy_fs = dummy_fs
+        for filename in filenames:
+            with MockSafeOpen(filename, framework="pytorch", dummy_fs=dummy_fs) as f:
+                for k in f.keys():
+                    if k in routing:
+                        raise RuntimeError(
+                            f"Key {k} was found in multiple files: {filename} and {routing[k]}"
+                        )
+                    routing[k] = filename
+        if aliases is None:
+            aliases = {}
+        self.aliases = aliases
+        self.routing = routing
+        self.device = device
+        self.dtype = dtype
+        self.process_group = process_group
+        self.prefix = prefix
+        self.weights_loader = (
+            # We don't need to get linear layers, so just wrap raw tensors.
+            DefaultWeightsLoader(lambda x: x)
+            if weights_loader is None
+            else weights_loader
+        )
+        self._handles = {}
+
+    def _get_handle(self, filename: Union[Path, str]):
+        if filename in self._handles:
+            return self._handles[filename]
+        else:
+            handle = mock_handle(filename, self.device, self.dtype)
+            self._handles[filename] = handle
+            return handle
+
+    def get_shape(self, tensor_name: str):
+        filename, _ = self.get_filename(tensor_name)
+        handle = self._get_handle(filename)
+        return handle.get_slice(tensor_name).get_shape()
+
+    def get_tensor(self, tensor_name: str):
+        filename, _ = self.get_filename(tensor_name)
+        handle = self._get_handle(filename)
+        return handle.get_slice(tensor_name).tensor
+
+
+dummy_process_group = SimpleNamespace(rank=lambda: 0, size=lambda: 1)
+
+
+def test_weights():
+    weights = MockWeights(
+        [
+            "test_weights",
+            "test_weights_2",
+        ],
+        device="cpu",
+        dtype=torch.float32,
+        process_group=dummy_process_group,
+        dummy_fs=dummy_file_system,
+    )
+    assert weights.get_shape("layer.0.weight") == (2, 2)
+    assert weights.get_tensor("layer.1337.weight").shape == (2, 4)
+
+
+def test_get_tensor():
+    weights = MockWeights(
+        [
+            "test_weights",
+            "test_weights_2",
+        ],
+        device="cpu",
+        dtype=torch.float32,
+        process_group=dummy_process_group,
+        dummy_fs=dummy_file_system,
+    )
+    assert torch.allclose(
+        weights.get_tensor("layer.0.weight"),
+        torch.tensor(
+            [
+                [1, 2],
+                [3, 4],
+            ],
+            dtype=torch.float32,
+        ),
+    )
+    assert torch.allclose(
+        weights.get_tensor("layer.1337.weight"),
+        torch.tensor(
+            [
+                [1, 2, 3, 4],
+                [5, 6, 7, 8],
+            ],
+            dtype=torch.float32,
+        ),
+    )
+
+
+def test_get_weights_col_packed():
+
+    weights = MockWeights(
+        [
+            "test_get_weights_col_packed",
+        ],
+        device="cpu",
+        dtype=torch.float32,
+        process_group=dummy_process_group,
+        dummy_fs=dummy_file_system,
+    )
+
+    prefix = "weight"
+    block_sizes = 1
+
+    w = weights.get_weights_col_packed(
+        prefix=prefix,
+        block_sizes=block_sizes,
+    )
+
+    assert torch.allclose(
+        w,
+        torch.tensor(
+            [
+                [1, 2],
+                [3, 4],
+                [5, 6],
+                [7, 8],
+            ],
+            dtype=torch.float32,
+        ),
+    )
+
+
+def test_get_weights_col_packed_block_size():
+
+    weights = MockWeights(
+        [
+            "test_get_weights_col_packed",
+        ],
+        device="cpu",
+        dtype=torch.float32,
+        process_group=dummy_process_group,
+        dummy_fs=dummy_file_system,
+    )
+
+    prefix = "weight"
+    block_sizes = 2
+
+    w = weights.get_weights_col_packed(
+        prefix=prefix,
+        block_sizes=block_sizes,
+    )
+
+    assert torch.allclose(
+        w,
+        torch.tensor(
+            [
+                [1, 2],
+                [3, 4],
+                [5, 6],
+                [7, 8],
+            ],
+            dtype=torch.float32,
+        ),
+    )
+
+
+def test_get_weights_col_packed_block_size_arr():
+
+    weights = MockWeights(
+        [
+            "test_get_weights_col_packed",
+        ],
+        device="cpu",
+        dtype=torch.float32,
+        process_group=dummy_process_group,
+        dummy_fs=dummy_file_system,
+    )
+
+    prefix = "weight"
+    block_sizes = [1, 1]
+
+    w = weights.get_weights_col_packed(
+        prefix=prefix,
+        block_sizes=block_sizes,
+    )
+
+    assert torch.allclose(
+        w,
+        torch.tensor(
+            [
+                [1, 2],
+                [3, 4],
+                [5, 6],
+                [7, 8],
+            ],
+            dtype=torch.float32,
+        ),
+    )
+
+
+def test_get_multi_weights_col():
+    weights = MockWeights(
+        [
+            "test_get_multi_weights_col",
+        ],
+        device="cpu",
+        dtype=torch.float32,
+        process_group=dummy_process_group,
+        dummy_fs=dummy_file_system,
+    )
+
+    prefixes = ["weight", "weight"]
+
+    w = weights.get_multi_weights_col(
+        prefixes=prefixes,
+        dim=0,
+    )
+
+    assert torch.allclose(
+        w,
+        torch.tensor(
+            [
+                [1, 2],
+                [3, 4],
+                [5, 6],
+                [7, 8],
+                [1, 2],
+                [3, 4],
+                [5, 6],
+                [7, 8],
+            ],
+            dtype=torch.float32,
+        ),
+    )
+
+
+def test_get_weights_row():
+    weights = MockWeights(
+        [
+            "test_get_weights_row",
+        ],
+        device="cpu",
+        dtype=torch.float32,
+        process_group=dummy_process_group,
+        dummy_fs=dummy_file_system,
+    )
+
+    prefix = "weight"
+
+    w = weights.get_weights_row(
+        prefix=prefix,
+    )
+
+    assert torch.allclose(
+        w,
+        torch.tensor(
+            [[1.0, 2.0], [3.0, 4.0], [5.0, 6.0], [7.0, 8.0]],
+            dtype=torch.float32,
+        ),
+    )
+
+
+# test_get_weights_col
+
+
+def test_get_weights_col_awq(gptq_weights_loader_awq):
+    weights = MockWeights(
+        [
+            "test_get_weights_col_gptq",
+        ],
+        device="cpu",
+        dtype=torch.float32,
+        process_group=dummy_process_group,
+        dummy_fs=dummy_file_system,
+        weights_loader=gptq_weights_loader_awq,
+    )
+
+    prefix = "weight"
+
+    w = weights.get_weights_col(
+        prefix=prefix,
+    )
+
+    expected_weight = GPTQWeight(
+        qweight=torch.tensor([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0], [7.0, 8.0]]),
+        qzeros=torch.tensor([[0, 1], [1, 0]], dtype=torch.int32),
+        scales=torch.tensor(
+            [[100.0, 100.0], [100.0, 100.0]],
+            dtype=torch.float16,
+        ),
+        g_idx=None,
+        bits=8.0,
+        groupsize=2.0,
+        use_awq_kernel=True,
+        use_exllama=False,
+    )
+
+    assert torch.allclose(w.qweight, expected_weight.qweight), "qweight mismatch"
+    assert torch.allclose(w.qzeros, expected_weight.qzeros), "qzeros mismatch"
+    assert torch.allclose(w.scales, expected_weight.scales), "scales mismatch"
+    assert w.g_idx == expected_weight.g_idx, "g_idx mismatch"
+    assert w.bits == expected_weight.bits, "bits mismatch"
+    assert w.groupsize == expected_weight.groupsize, "groupsize mismatch"
+    assert w.use_awq_kernel == expected_weight.use_awq_kernel, "use_awq_kernel mismatch"
+    assert w.use_exllama == expected_weight.use_exllama, "use_exllama mismatch"
+
+
+def test_get_weights_col_gtpq(gptq_weights_loader):
+    weights = MockWeights(
+        [
+            "test_get_weights_col_gptq",
+        ],
+        device="cpu",
+        dtype=torch.float32,
+        process_group=dummy_process_group,
+        dummy_fs=dummy_file_system,
+        weights_loader=gptq_weights_loader,
+    )
+
+    prefix = "weight"
+
+    w = weights.get_weights_col(
+        prefix=prefix,
+    )
+
+    expected_weight = GPTQWeight(
+        qweight=torch.tensor([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0], [7.0, 8.0]]),
+        qzeros=torch.tensor([[0, 1], [1, 0]], dtype=torch.int32),
+        scales=torch.tensor([[100.0, 100.0], [100.0, 100.0]], dtype=torch.float16),
+        g_idx=torch.tensor([0, 1, 0, 1], dtype=torch.int32),
+        bits=8.0,
+        groupsize=2.0,
+        use_awq_kernel=False,
+        use_exllama=False,
+    )
+
+    assert torch.allclose(w.qweight, expected_weight.qweight), "qweight mismatch"
+    assert torch.allclose(w.qzeros, expected_weight.qzeros), "qzeros mismatch"
+    assert torch.allclose(w.scales, expected_weight.scales), "scales mismatch"
+    assert torch.allclose(w.g_idx, expected_weight.g_idx), "g_idx mismatch"
+    assert w.bits == expected_weight.bits, "bits mismatch"
+    assert w.groupsize == expected_weight.groupsize, "groupsize mismatch"
+    assert w.use_awq_kernel == expected_weight.use_awq_kernel, "use_awq_kernel mismatch"
+    assert w.use_exllama == expected_weight.use_exllama, "use_exllama mismatch"
+
+
+def test_get_weights_col_exl2():
+    weights = MockWeights(
+        [
+            "test_get_weights_col_exl2",
+        ],
+        device="cpu",
+        dtype=torch.float32,
+        process_group=dummy_process_group,
+        dummy_fs=dummy_file_system,
+        weights_loader=Exl2WeightsLoader(),
+    )
+
+    prefix = "weight"
+
+    w = weights.get_weights_col(
+        prefix=prefix,
+    )
+
+    scaled_scale_max = 0.3906 * 256
+    expected_weight = Exl2Weight(
+        q_weight=torch.tensor([[1, 2], [3, 4], [5, 6], [7, 8]], dtype=torch.int32),
+        q_scale=torch.tensor([8], dtype=torch.int32),
+        q_invperm=torch.tensor([1, 0, 3, 2], dtype=torch.int16),
+        q_scale_max=torch.tensor([scaled_scale_max], dtype=torch.float16),
+        q_groups=torch.tensor([4], dtype=torch.int16),
+    )
+
+    assert torch.allclose(w.q_weight, expected_weight.q_weight), "q_weight mismatch"
+    assert torch.allclose(w.q_scale, expected_weight.q_scale), "q_scale mismatch"
+    assert torch.allclose(w.q_invperm, expected_weight.q_invperm), "q_invperm mismatch"
+    assert torch.allclose(
+        w.q_scale_max, expected_weight.q_scale_max
+    ), "q_scale_max mismatch"
+    assert torch.allclose(w.q_groups, expected_weight.q_groups), "q_groups mismatch"
+
+
+def test_get_weights_col_marlin(marlin_weights_loader):
+    weights = MockWeights(
+        [
+            "test_get_weights_col_marlin",
+        ],
+        device="cpu",
+        dtype=torch.float16,
+        process_group=dummy_process_group,
+        dummy_fs=dummy_file_system,
+        weights_loader=marlin_weights_loader,
+    )
+
+    prefix = "weight"
+
+    w = weights.get_weights_col(
+        prefix=prefix,
+    )
+
+    expected_weight = MarlinWeight(
+        B=torch.tensor([[1, 2], [3, 4]], dtype=torch.int32),
+        s=torch.tensor([[0.5000], [0.2500]], dtype=torch.float16),
+    )
+
+    assert torch.allclose(w.B, expected_weight.B), "B mismatch"
+    assert torch.allclose(w.s, expected_weight.s), "s mismatch"
+
+
+# test_get_weights_col_packed
+
+
+def test_get_weights_col_packed_awq(gptq_weights_loader_awq):
+    weights = MockWeights(
+        [
+            "test_get_weights_col_packed_gptq",
+        ],
+        device="cpu",
+        dtype=torch.float32,
+        process_group=dummy_process_group,
+        dummy_fs=dummy_file_system,
+        weights_loader=gptq_weights_loader_awq,
+    )
+
+    prefix = "weight"
+    block_sizes = 1
+
+    w = weights.get_weights_col_packed(
+        prefix=prefix,
+        block_sizes=block_sizes,
+    )
+
+    expected_weight = GPTQWeight(
+        qweight=torch.tensor([[1, 2], [3, 4], [5, 6], [7, 8]], dtype=torch.int32),
+        qzeros=torch.tensor([[0, 1], [1, 0]], dtype=torch.int32),
+        scales=torch.tensor([[100.0, 100.0], [100.0, 100.0]], dtype=torch.float16),
+        g_idx=None,
+        bits=8.0,
+        groupsize=2.0,
+        use_awq_kernel=True,
+        use_exllama=False,
+    )
+
+    assert torch.allclose(w.qweight, expected_weight.qweight), "qweight mismatch"
+    assert torch.allclose(w.qzeros, expected_weight.qzeros), "qzeros mismatch"
+    assert torch.allclose(w.scales, expected_weight.scales), "scales mismatch"
+    assert w.g_idx == expected_weight.g_idx, "g_idx mismatch"
+    assert w.bits == expected_weight.bits, "bits mismatch"
+    assert w.groupsize == expected_weight.groupsize, "groupsize mismatch"
+    assert w.use_awq_kernel == expected_weight.use_awq_kernel, "use_awq_kernel mismatch"
+    assert w.use_exllama == expected_weight.use_exllama, "use_exllama mismatch"
+
+
+@pytest.mark.skip(reason="Review expected functionality")
+def test_get_weights_col_packed_exl2():
+    weights = MockWeights(
+        [
+            "test_get_weights_col_packed_exl2",
+        ],
+        device="cpu",
+        dtype=torch.float32,
+        process_group=dummy_process_group,
+        dummy_fs=dummy_file_system,
+        weights_loader=Exl2WeightsLoader(),
+    )
+
+    prefix = "weight"
+    block_sizes = 1
+
+    w = weights.get_weights_col_packed(
+        prefix=prefix,
+        block_sizes=block_sizes,
+    )
+
+    scaled_scale_max = 0.3906 * 256
+    expected_weight = Exl2Weight(
+        q_weight=torch.tensor([[1, 2], [3, 4], [5, 6], [7, 8]], dtype=torch.int32),
+        q_scale=torch.tensor([8], dtype=torch.int32),
+        q_invperm=torch.tensor([1], dtype=torch.int16),
+        q_scale_max=torch.tensor([scaled_scale_max], dtype=torch.float16),
+        q_groups=torch.tensor([4], dtype=torch.int16),
+    )
+
+    assert torch.allclose(w.q_weight, expected_weight.q_weight), "q_weight mismatch"
+    assert torch.allclose(w.q_scale, expected_weight.q_scale), "q_scale mismatch"
+    assert torch.allclose(w.q_invperm, expected_weight.q_invperm), "q_invperm mismatch"
+    assert torch.allclose(
+        w.q_scale_max, expected_weight.q_scale_max
+    ), "q_scale_max mismatch"
+    assert torch.allclose(w.q_groups, expected_weight.q_groups), "q_groups mismatch"
+
+
+def test_get_weights_col_packed_gptq(gptq_weights_loader):
+    weights = MockWeights(
+        [
+            "test_get_weights_col_packed_gptq",
+        ],
+        device="cpu",
+        dtype=torch.float32,
+        process_group=dummy_process_group,
+        dummy_fs=dummy_file_system,
+        weights_loader=gptq_weights_loader,
+    )
+
+    prefixes = ["weight"]
+
+    w = weights.get_multi_weights_col(
+        prefixes=prefixes,
+        dim=0,
+    )
+
+    expected_weight = GPTQWeight(
+        qweight=torch.tensor([[1, 2], [3, 4], [5, 6], [7, 8]], dtype=torch.int32),
+        qzeros=torch.tensor([[0, 1], [1, 0]], dtype=torch.int32),
+        scales=torch.tensor([[100.0, 100.0], [100.0, 100.0]], dtype=torch.float16),
+        g_idx=torch.tensor([0, 1, 0, 1], dtype=torch.int32),
+        bits=8.0,
+        groupsize=2.0,
+        use_awq_kernel=False,
+        use_exllama=False,
+    )
+
+    assert torch.allclose(w.qweight, expected_weight.qweight), "qweight mismatch"
+    assert torch.allclose(w.qzeros, expected_weight.qzeros), "qzeros mismatch"
+    assert torch.allclose(w.scales, expected_weight.scales), "scales mismatch"
+    assert torch.allclose(w.g_idx, expected_weight.g_idx), "g_idx mismatch"
+    assert w.bits == expected_weight.bits, "bits mismatch"
+    assert w.groupsize == expected_weight.groupsize, "groupsize mismatch"
+    assert w.use_awq_kernel == expected_weight.use_awq_kernel, "use_awq_kernel mismatch"
+    assert w.use_exllama == expected_weight.use_exllama, "use_exllama mismatch"
+
+
+def test_get_weights_col_packed_marlin(marlin_weights_loader):
+    weights = MockWeights(
+        [
+            "test_get_weights_col_packed_marlin",
+        ],
+        device="cpu",
+        dtype=torch.float16,
+        process_group=dummy_process_group,
+        dummy_fs=dummy_file_system,
+        weights_loader=marlin_weights_loader,
+    )
+
+    prefix = "weight"
+
+    w = weights.get_multi_weights_col(
+        prefixes=[prefix],
+        dim=0,
+    )
+
+    expected_weight = MarlinWeight(
+        B=torch.tensor([[1, 2], [3, 4]], dtype=torch.int32),
+        s=torch.tensor([[0.5000], [0.2500]], dtype=torch.float16),
+    )
+
+    print(expected_weight)
+
+    assert torch.allclose(w.B, expected_weight.B), "B mismatch"
+    assert torch.allclose(w.s, expected_weight.s), "s mismatch"
+
+
+# test_get_multi_weights_col
+
+
+def test_get_multi_weights_col_awq(gptq_weights_loader_awq):
+    weights = MockWeights(
+        [
+            "test_get_multi_weights_col_gptq",
+        ],
+        device="cpu",
+        dtype=torch.float32,
+        process_group=dummy_process_group,
+        dummy_fs=dummy_file_system,
+        weights_loader=gptq_weights_loader_awq,
+    )
+
+    prefixes = ["weight"]
+
+    w = weights.get_multi_weights_col(
+        prefixes=prefixes,
+        dim=0,
+    )
+
+    expected_weight = GPTQWeight(
+        qweight=torch.tensor([[1, 2], [3, 4], [5, 6], [7, 8]], dtype=torch.int32),
+        qzeros=torch.tensor([[0, 1], [1, 0]], dtype=torch.int32),
+        scales=torch.tensor([[100.0, 100.0], [100.0, 100.0]], dtype=torch.float16),
+        g_idx=None,
+        bits=8.0,
+        groupsize=2.0,
+        use_awq_kernel=True,
+        use_exllama=False,
+    )
+
+    assert torch.allclose(w.qweight, expected_weight.qweight), "qweight mismatch"
+    assert torch.allclose(w.qzeros, expected_weight.qzeros), "qzeros mismatch"
+    assert torch.allclose(w.scales, expected_weight.scales), "scales mismatch"
+    assert w.g_idx == expected_weight.g_idx, "g_idx mismatch"
+    assert w.bits == expected_weight.bits, "bits mismatch"
+    assert w.groupsize == expected_weight.groupsize, "groupsize mismatch"
+    assert w.use_awq_kernel == expected_weight.use_awq_kernel, "use_awq_kernel mismatch"
+    assert w.use_exllama == expected_weight.use_exllama, "use_exllama mismatch"
+
+
+def test_get_multi_weights_col_exl2():
+    weights = MockWeights(
+        [
+            "test_get_multi_weights_col_exl2",
+        ],
+        device="cpu",
+        dtype=torch.float32,
+        process_group=dummy_process_group,
+        dummy_fs=dummy_file_system,
+        weights_loader=Exl2WeightsLoader(),
+    )
+
+    prefix = "weight"
+
+    try:
+        weights.get_multi_weights_col(
+            prefixes=[prefix],
+            dim=0,
+        )
+    except ValueError as e:
+        assert e.args[0] == "get_multi_weights_col is not supported for exl2"
+
+
+def test_get_multi_weights_col_gptq(gptq_weights_loader):
+    weights = MockWeights(
+        [
+            "test_get_multi_weights_col_gptq",
+        ],
+        device="cpu",
+        dtype=torch.float32,
+        process_group=dummy_process_group,
+        dummy_fs=dummy_file_system,
+        weights_loader=gptq_weights_loader,
+    )
+
+    prefixes = ["weight"]
+
+    w = weights.get_multi_weights_col(
+        prefixes=prefixes,
+        dim=0,
+    )
+
+    expected_weight = GPTQWeight(
+        qweight=torch.tensor([[1, 2], [3, 4], [5, 6], [7, 8]], dtype=torch.int32),
+        qzeros=torch.tensor([[0, 1], [1, 0]], dtype=torch.int32),
+        scales=torch.tensor([[100.0, 100.0], [100.0, 100.0]], dtype=torch.float16),
+        g_idx=torch.tensor([0, 1, 0, 1], dtype=torch.int32),
+        bits=8.0,
+        groupsize=2.0,
+        use_awq_kernel=False,
+        use_exllama=False,
+    )
+
+    assert torch.allclose(w.qweight, expected_weight.qweight), "qweight mismatch"
+    assert torch.allclose(w.qzeros, expected_weight.qzeros), "qzeros mismatch"
+    assert torch.allclose(w.scales, expected_weight.scales), "scales mismatch"
+    assert torch.allclose(w.g_idx, expected_weight.g_idx), "g_idx mismatch"
+    assert w.bits == expected_weight.bits, "bits mismatch"
+    assert w.groupsize == expected_weight.groupsize, "groupsize mismatch"
+    assert w.use_awq_kernel == expected_weight.use_awq_kernel, "use_awq_kernel mismatch"
+    assert w.use_exllama == expected_weight.use_exllama, "use_exllama mismatch"
+
+
+def test_get_multi_weights_col_marlin(marlin_weights_loader):
+    weights = MockWeights(
+        [
+            "test_get_multi_weights_col_marlin",
+        ],
+        device="cpu",
+        dtype=torch.float16,
+        process_group=dummy_process_group,
+        dummy_fs=dummy_file_system,
+        weights_loader=marlin_weights_loader,
+    )
+
+    prefix = "weight"
+
+    w = weights.get_multi_weights_col(
+        prefixes=[prefix],
+        dim=0,
+    )
+
+    expected_weight = MarlinWeight(
+        B=torch.tensor([[1, 2], [3, 4]], dtype=torch.int32),
+        s=torch.tensor([[0.5000], [0.2500]], dtype=torch.float16),
+    )
+
+    assert torch.allclose(w.B, expected_weight.B), "B mismatch"
+    assert torch.allclose(w.s, expected_weight.s), "s mismatch"
+
+
+# test_get_weights_row
+
+
+def test_get_weights_row_awq(gptq_weights_loader_awq):
+    weights = MockWeights(
+        [
+            "test_get_weights_row_gptq",
+        ],
+        device="cpu",
+        dtype=torch.float32,
+        process_group=dummy_process_group,
+        dummy_fs=dummy_file_system,
+        weights_loader=gptq_weights_loader_awq,
+    )
+
+    prefix = "weight"
+
+    w = weights.get_weights_row(
+        prefix=prefix,
+    )
+
+    expected_weight = GPTQWeight(
+        qweight=torch.tensor([[1, 2], [3, 4], [5, 6], [7, 8]], dtype=torch.int32),
+        qzeros=torch.tensor([[0, 1], [1, 0]], dtype=torch.int32),
+        scales=torch.tensor([[100.0, 100.0], [100.0, 100.0]], dtype=torch.float16),
+        g_idx=None,
+        bits=8.0,
+        groupsize=2.0,
+        use_awq_kernel=True,
+        use_exllama=False,
+    )
+
+    assert torch.allclose(w.qweight, expected_weight.qweight), "qweight mismatch"
+    assert torch.allclose(w.qzeros, expected_weight.qzeros), "qzeros mismatch"
+    assert torch.allclose(w.scales, expected_weight.scales), "scales mismatch"
+    assert w.g_idx == expected_weight.g_idx, "g_idx mismatch"
+    assert w.bits == expected_weight.bits, "bits mismatch"
+    assert w.groupsize == expected_weight.groupsize, "groupsize mismatch"
+    assert w.use_awq_kernel == expected_weight.use_awq_kernel, "use_awq_kernel mismatch"
+    assert w.use_exllama == expected_weight.use_exllama, "use_exllama mismatch"
+
+
+def test_get_weights_row_exl2():
+    weights = MockWeights(
+        [
+            "test_get_weights_row_exl2",
+        ],
+        device="cpu",
+        dtype=torch.float32,
+        process_group=dummy_process_group,
+        dummy_fs=dummy_file_system,
+        weights_loader=Exl2WeightsLoader(),
+    )
+
+    prefix = "weight"
+
+    w = weights.get_weights_row(
+        prefix=prefix,
+    )
+    print(w)
+
+    scaled_scale_max = 0.3906 * 256
+    expected_weight = Exl2Weight(
+        q_weight=torch.tensor([[1, 2], [3, 4], [5, 6], [7, 8]], dtype=torch.int32),
+        q_scale=torch.tensor([8], dtype=torch.int32),
+        q_invperm=torch.tensor([1, 0, 3, 2], dtype=torch.int16),
+        q_scale_max=torch.tensor([scaled_scale_max], dtype=torch.float16),
+        q_groups=torch.tensor([4], dtype=torch.int16),
+    )
+
+    assert torch.allclose(w.q_weight, expected_weight.q_weight), "q_weight mismatch"
+    assert torch.allclose(w.q_scale, expected_weight.q_scale), "q_scale mismatch"
+    assert torch.allclose(w.q_invperm, expected_weight.q_invperm), "q_invperm mismatch"
+    assert torch.allclose(
+        w.q_scale_max, expected_weight.q_scale_max
+    ), "q_scale_max mismatch"
+    assert torch.allclose(w.q_groups, expected_weight.q_groups), "q_groups mismatch"
+
+
+def test_get_weights_row_gptq(gptq_weights_loader):
+    weights = MockWeights(
+        [
+            "test_get_weights_row_gptq",
+        ],
+        device="cpu",
+        dtype=torch.float32,
+        process_group=dummy_process_group,
+        dummy_fs=dummy_file_system,
+        weights_loader=gptq_weights_loader,
+    )
+
+    prefix = "weight"
+
+    w = weights.get_weights_row(
+        prefix=prefix,
+    )
+
+    expected_weight = GPTQWeight(
+        qweight=torch.tensor([[1, 2], [3, 4], [5, 6], [7, 8]], dtype=torch.int32),
+        qzeros=torch.tensor([[0, 1], [1, 0]], dtype=torch.int32),
+        scales=torch.tensor([[100.0, 100.0], [100.0, 100.0]], dtype=torch.float16),
+        g_idx=torch.tensor([0, 1, 0, 1], dtype=torch.int32),
+        bits=8.0,
+        groupsize=2.0,
+        use_awq_kernel=False,
+        use_exllama=False,
+    )
+
+    assert torch.allclose(w.qweight, expected_weight.qweight), "qweight mismatch"
+    assert torch.allclose(w.qzeros, expected_weight.qzeros), "qzeros mismatch"
+    assert torch.allclose(w.scales, expected_weight.scales), "scales mismatch"
+    assert torch.allclose(w.g_idx, expected_weight.g_idx), "g_idx mismatch"
+    assert w.bits == expected_weight.bits, "bits mismatch"
+    assert w.groupsize == expected_weight.groupsize, "groupsize mismatch"
+    assert w.use_awq_kernel == expected_weight.use_awq_kernel, "use_awq_kernel mismatch"
+    assert w.use_exllama == expected_weight.use_exllama, "use_exllama mismatch"
+
+
+def test_get_weights_row_marlin(marlin_weights_loader):
+    weights = MockWeights(
+        [
+            "test_get_weights_row_marlin",
+        ],
+        device="cpu",
+        dtype=torch.float16,
+        process_group=dummy_process_group,
+        dummy_fs=dummy_file_system,
+        weights_loader=marlin_weights_loader,
+    )
+
+    prefix = "weight"
+
+    w = weights.get_weights_row(
+        prefix=prefix,
+    )
+
+    expected_weight = MarlinWeight(
+        B=torch.tensor([[1, 2], [3, 4]], dtype=torch.int32),
+        s=torch.tensor([[0.5000], [0.2500]], dtype=torch.float16),
+    )
+
+    assert torch.allclose(w.B, expected_weight.B), "B mismatch"
+    assert torch.allclose(w.s, expected_weight.s), "s mismatch"
diff --git a/server/text_generation_server/adapters/__init__.py b/server/text_generation_server/adapters/__init__.py
new file mode 100644
index 00000000..8697cb9e
--- /dev/null
+++ b/server/text_generation_server/adapters/__init__.py
@@ -0,0 +1,13 @@
+# Origin:   https://github.com/predibase/lorax
+# Path:     lorax/server/lorax_server/adapters/__init__.py
+# License:  Apache License Version 2.0, January 2004
+
+from text_generation_server.adapters.weights import (
+    AdapterBatchData,
+    AdapterBatchMetadata,
+)
+
+__all__ = [
+    "AdapterBatchData",
+    "AdapterBatchMetadata",
+]
diff --git a/server/text_generation_server/adapters/config.py b/server/text_generation_server/adapters/config.py
new file mode 100644
index 00000000..b7e27090
--- /dev/null
+++ b/server/text_generation_server/adapters/config.py
@@ -0,0 +1,30 @@
+# Origin:   https://github.com/predibase/lorax
+# Path:     lorax/server/lorax_server/adapters/config.py
+# License:  Apache License Version 2.0, January 2004
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Dict, Set, Tuple
+
+import torch
+
+from text_generation_server.adapters.weights import AdapterWeights
+
+
+@dataclass
+class ModuleMap:
+    module_name: str
+    module_weights: Dict[str, Tuple[torch.Tensor, str]]
+
+
+@dataclass
+class AdapterConfig(ABC):
+    base_model_name_or_path: str
+
+    @abstractmethod
+    def map_weights_for_model(
+        self,
+        adapter_weights: Dict[int, AdapterWeights],
+        weight_names: Tuple[str],
+    ) -> Tuple[ModuleMap, Set[str]]:
+        pass
diff --git a/server/text_generation_server/adapters/lora.py b/server/text_generation_server/adapters/lora.py
new file mode 100644
index 00000000..a00338e7
--- /dev/null
+++ b/server/text_generation_server/adapters/lora.py
@@ -0,0 +1,471 @@
+# Origin:   https://github.com/predibase/lorax
+# Path:     lorax/server/lorax_server/adapters/lora.py
+# License:  Apache License Version 2.0, January 2004
+
+from collections import defaultdict
+from dataclasses import dataclass
+from typing import Dict, List, Optional, Set, Tuple, Type, Union
+
+import torch
+from peft import LoraConfig as _LoraConfig
+from torch.distributed import ProcessGroup
+
+from text_generation_server.adapters.config import AdapterConfig, ModuleMap
+
+from text_generation_server.adapters.weights import (
+    AdapterBatchMetadata,
+    AdapterWeights,
+    BatchAdapterWeights,
+)
+from text_generation_server.utils.sgmv import (
+    BGMV_MAX_RANK,
+    MAX_RANK_CUSTOM,
+    get_tmp_tensors,
+    orient_for_rank,
+    pad_rank,
+    use_cutlass_shrink,
+)
+
+
+def get_start_stop_idxs_for_rank(offset, size, rank, world_size):
+    block_size = size // world_size
+    start = offset + rank * block_size
+    stop = offset + (rank + 1) * block_size
+    return start, stop
+
+
+def shard_on_dim(
+    t: torch.Tensor, dim: int, process_group: torch.distributed.ProcessGroup
+):
+    world_size = process_group.size()
+    rank = process_group.rank()
+
+    size = t.shape[dim]
+    start, stop = get_start_stop_idxs_for_rank(0, size, rank, world_size)
+
+    if dim == 0:
+        tensor = t[start:stop]
+    elif dim == 1:
+        tensor = t[:, start:stop]
+    else:
+        raise NotImplementedError("Let's make that generic when needed")
+
+    return tensor
+
+
+def shard_lora_weights(
+    weights_a: List[torch.Tensor],
+    weights_b: List[torch.Tensor],
+    split_dim: int,
+    process_group: ProcessGroup,
+) -> Tuple[List[torch.Tensor], List[torch.Tensor]]:
+    # [hidden_size, r]
+    weights_a = [
+        shard_on_dim(w, dim=split_dim, process_group=process_group) for w in weights_a
+    ]
+
+    # [r, hidden_size]
+    weights_b = [shard_on_dim(w, dim=1, process_group=process_group) for w in weights_b]
+
+    return weights_a, weights_b
+
+
+@dataclass
+class LoraConfig(AdapterConfig):
+    r: int
+    target_modules: Optional[Union[List[str], str]]
+    fan_in_fan_out: bool
+    lora_alpha: int
+    use_rslora: bool
+
+    def map_weights_for_model(
+        self,
+        adapter_weights: Dict[int, AdapterWeights],
+        weight_names: Tuple[str],
+    ) -> Tuple[ModuleMap, Set[str]]:
+        adapter_weight_names = set()
+        module_map = {}
+        for weight_name in weight_names:
+            lora_a_name = f"base_model.model.{weight_name}.lora_A.weight"
+            lora_b_name = f"base_model.model.{weight_name}.lora_B.weight"
+            if lora_a_name not in adapter_weights or lora_b_name not in adapter_weights:
+                continue
+
+            module_map[weight_name] = {
+                "lora_A": (adapter_weights[lora_a_name], lora_a_name),
+                "lora_B": (adapter_weights[lora_b_name], lora_b_name),
+            }
+            adapter_weight_names.add(lora_a_name)
+            adapter_weight_names.add(lora_b_name)
+        return module_map, adapter_weight_names
+
+    @classmethod
+    def load(cls, adapter_id: str, api_token: str) -> "LoraConfig":
+        hf_config = _LoraConfig.from_pretrained(adapter_id, token=api_token)
+        return cls(
+            base_model_name_or_path=hf_config.base_model_name_or_path,
+            r=hf_config.r,
+            target_modules=hf_config.target_modules,
+            fan_in_fan_out=hf_config.fan_in_fan_out,
+            lora_alpha=hf_config.lora_alpha,
+            use_rslora=(
+                hf_config.use_rslora if hasattr(hf_config, "use_rslora") else False
+            ),
+        )
+
+
+class LoraWeights(AdapterWeights):
+    """LoRA weights for a single adapter merged across all layers."""
+
+    def __init__(
+        self,
+        weights_a: List[torch.Tensor],
+        weights_b: List[torch.Tensor],
+        adapter_config: LoraConfig,
+    ):
+        self.lora_a_r = weights_a[0].size(1) if len(weights_a) > 0 else 1
+        self.lora_b_r = weights_b[0].size(0) if len(weights_a) > 0 else 1
+
+        self._use_cutlass_shrink = use_cutlass_shrink(self.lora_a_r)
+        self._is_transposed = False
+
+        # [num_layers, hidden_size, r]
+        weights_a = [orient_for_rank(w, w.size(1)).contiguous() for w in weights_a]
+        self._weights_a = torch.stack(weights_a)
+
+        # [num_layers, r, hidden_size]
+        self._weights_b = torch.stack(weights_b)
+
+        self.adapter_config = adapter_config
+
+    @property
+    def weights_a(self) -> torch.Tensor:
+        if self._is_transposed:
+            self._transpose_weights()
+        return self._weights_a
+
+    @property
+    def weights_b(self) -> torch.Tensor:
+        if self._is_transposed:
+            self._transpose_weights()
+        return self._weights_b
+
+    @property
+    def weights_a_t(self) -> torch.Tensor:
+        if not self._is_transposed:
+            self._transpose_weights()
+        return self._weights_a
+
+    @property
+    def weights_b_t(self) -> torch.Tensor:
+        if not self._is_transposed:
+            self._transpose_weights()
+        return self._weights_b
+
+    def _transpose_weights(self):
+        if self._use_cutlass_shrink:
+            # If we're not using the cutlass shrink, then both SGMV and BGMV use the same orientation
+            self._weights_a = self._weights_a.transpose(1, 2).contiguous()
+        self._weights_b = self._weights_b.transpose(1, 2).contiguous()
+        self._is_transposed = not self._is_transposed
+
+    @classmethod
+    def get_batch_types(cls) -> List[Type[BatchAdapterWeights]]:
+        return [BatchLoraWeights]
+
+    # prepare pre-loaded lora weights for use in the model.
+    #
+    # this method processes and organizes lora weights for a specific layer type across all layers:
+    # - uses `config` (LoraConfig) to apply lora-specific settings like scaling factor.
+    # - retrieves weights from `module_map` based on the `layer_type`.
+    # - processes `nlayers` number of layers.
+    # - converts weights to the specified `dtype`.
+    # - shards weights across `world_size` number of processes using the `process_group`.
+    # - maps weights to specific layers using `target_to_layer`.
+    # - tracks `unused_weight_names` to identify any unused weights.
+    #
+    # the method handles weight transposition, scaling, and padding to ensure compatibility
+    # with SGMV or BGMV operations.
+    @classmethod
+    def prepare_weights(
+        cls,
+        config: LoraConfig,
+        module_map: Dict[str, Dict],
+        layer_type: str,
+        unused_weight_names: Set[str],
+        nlayers: int,
+        dtype: torch.dtype,
+        world_size: int,
+        process_group: ProcessGroup,
+        target_to_layer: Dict[str, Tuple[str, torch.Tensor]],
+    ) -> Optional[AdapterWeights]:
+        lora_a_list = [None] * nlayers
+        lora_b_list = [None] * nlayers
+
+        for layer_id in range(nlayers):
+            key = (layer_id, layer_type)
+            weight_name, layer = target_to_layer[key]
+            base_weight = layer.base_layer.linear.weight
+            base_device = base_weight.device
+
+            if weight_name not in module_map:
+                # There is no LoRA weight for this layer type in the adapter
+                return None
+
+            lora_a, lora_a_name = module_map[weight_name]["lora_A"]
+            lora_a = lora_a.to(base_device, dtype)
+
+            lora_b, lora_b_name = module_map[weight_name]["lora_B"]
+            lora_b = lora_b.to(base_device, dtype)
+
+            scale = get_scaling_factor(
+                config.lora_alpha,
+                config.r,
+                uses_rslora=config.use_rslora,
+            )
+
+            unused_weight_names.discard(lora_a_name)
+            unused_weight_names.discard(lora_b_name)
+
+            # Merge scaling factor into lora_b due to associativity of matrix multiplication:
+            # (A * B) * C = A * (B * C)
+            lora_a_list[layer_id] = lora_a.transpose(0, 1)
+            lora_b_list[layer_id] = lora_b.transpose(0, 1) * scale
+
+        # pad lora ranks to be compatible with sgmv
+        lora_a_list = [pad_rank(w, dim=1, world_size=world_size) for w in lora_a_list]
+        lora_b_list = [pad_rank(w, dim=0, world_size=world_size) for w in lora_b_list]
+
+        if lora_a_list:
+            # update rank if it was padded
+            padded_rank = lora_a_list[0].size(1)
+            config.r = padded_rank
+
+        return LoraWeights(
+            *shard_lora_weights(
+                weights_a=lora_a_list,
+                weights_b=lora_b_list,
+                split_dim=0 if layer_type in {"o_proj", "down_proj", "lm_head"} else 1,
+                process_group=process_group,
+            ),
+            config,
+        )
+
+
+@dataclass
+class RankSegments:
+    rank: int
+
+    lora_a_ptr: torch.Tensor
+    lora_b_ptr: torch.Tensor
+
+    # prefill (sgmv)
+    tmp_shrink: torch.Tensor
+    tmp_expand: torch.Tensor
+    segment_starts: torch.Tensor
+    segment_ends: torch.Tensor
+
+    # decode (bgmv)
+    indices: torch.Tensor
+
+
+@dataclass
+class BatchLoraWeights(BatchAdapterWeights):
+    lora_a: Dict[int, torch.Tensor]
+    lora_b: Dict[int, torch.Tensor]
+    adapter_index_configs: Dict[int, LoraConfig]
+    rank_data: Dict[int, RankSegments]
+    use_sgmv: bool
+
+    def has_adapter(self, adapter_index: int) -> bool:
+        return adapter_index in self.adapter_index_configs
+
+    def can_vectorize(self, pg: ProcessGroup) -> bool:
+        return all(
+            rank_data.rank // pg.size() <= MAX_RANK_CUSTOM
+            for rank_data in self.rank_data.values()
+        )
+
+    @classmethod
+    def load(
+        self,
+        adapter_weights: Dict[int, AdapterWeights],
+        meta: AdapterBatchMetadata,
+        prefill: bool,
+        prefill_head_indices: Optional[torch.Tensor],
+    ) -> Optional["BatchLoraWeights"]:
+        adapter_weights = {k: _convert_lora(v) for k, v in adapter_weights.items()}
+        adapter_weights = {
+            k: v for k, v in adapter_weights.items() if isinstance(v, LoraWeights)
+        }
+        if not adapter_weights:
+            return None
+
+        first_weights = next(iter(adapter_weights.values()))
+        device = first_weights.weights_a.device
+        segment_indices = meta.segment_indices
+
+        lora_a = {
+            idx: adapter_weights[idx].weights_a
+            for idx in segment_indices
+            if idx in adapter_weights
+        }
+        lora_b = {
+            idx: adapter_weights[idx].weights_b
+            for idx in segment_indices
+            if idx in adapter_weights
+        }
+
+        max_rank = max(
+            (
+                adapter_weights[idx].lora_a_r
+                for idx in segment_indices
+                if idx in adapter_weights
+            ),
+            default=0,
+        )
+
+        if prefill or max_rank > BGMV_MAX_RANK:
+            use_sgmv = True
+            lora_a_ptr = torch.tensor(
+                [
+                    (
+                        adapter_weights[idx].weights_a.data_ptr()
+                        if idx in adapter_weights
+                        else 0
+                    )
+                    for idx in segment_indices
+                ],
+                dtype=torch.int64,
+                device=device,
+            )
+            lora_b_ptr = torch.tensor(
+                [
+                    (
+                        adapter_weights[idx].weights_b.data_ptr()
+                        if idx in adapter_weights
+                        else 0
+                    )
+                    for idx in segment_indices
+                ],
+                dtype=torch.int64,
+                device=device,
+            )
+        else:
+            use_sgmv = False
+            lora_a_ptr = torch.tensor(
+                [
+                    (
+                        adapter_weights[idx].weights_a_t.data_ptr()
+                        if idx in adapter_weights
+                        else 0
+                    )
+                    for idx in segment_indices
+                ],
+                dtype=torch.int64,
+                device=device,
+            )
+            lora_b_ptr = torch.tensor(
+                [
+                    (
+                        adapter_weights[idx].weights_b_t.data_ptr()
+                        if idx in adapter_weights
+                        else 0
+                    )
+                    for idx in segment_indices
+                ],
+                dtype=torch.int64,
+                device=device,
+            )
+
+        adapter_index_configs = {
+            idx: adapter_weights[idx].adapter_config
+            for idx in segment_indices
+            if idx in adapter_weights
+        }
+
+        adapter_to_segment = {v: k for k, v in enumerate(segment_indices)}
+
+        rank_indices = defaultdict(list)
+        for segment_idx, adapter_idx in enumerate(segment_indices):
+            if adapter_idx not in adapter_weights:
+                continue
+            rank_indices[adapter_weights[adapter_idx].lora_a_r].append(segment_idx)
+
+        if prefill_head_indices is not None:
+            j, prefill_head_segment_starts, prefill_head_segment_ends = 1, [0], [0]
+            for head_index in prefill_head_indices:
+                # j cannot go out of bounds as that would mean there are tokens without corresponding adapters
+                if head_index < meta.adapter_segments[j]:
+                    prefill_head_segment_ends[-1] += 1
+                else:
+                    prefill_head_segment_starts.append(prefill_head_segment_ends[-1])
+                    prefill_head_segment_ends.append(prefill_head_segment_ends[-1] + 1)
+                    j += 1
+
+        rank_data = {}
+        for rank, indices in rank_indices.items():
+            tmp_shrink = None
+            tmp_expand = None
+            segment_starts = None
+            segment_ends = None
+            batch_indices = None
+
+            if use_sgmv:
+                lora_a_ptr_indices = lora_a_ptr[indices]
+                tmp_shrink, tmp_expand = get_tmp_tensors(
+                    lora_a_ptr_indices.size(0), rank, device
+                )
+                segment_starts = meta.adapter_segments[indices]
+                segment_ends = meta.adapter_segments[[i + 1 for i in indices]]
+                if prefill_head_indices is not None:
+                    for i, segment_index in enumerate(indices):
+                        segment_starts[i] = prefill_head_segment_starts[segment_index]
+                        segment_ends[i] = prefill_head_segment_ends[segment_index]
+            else:
+                rank_indices = set(indices)
+                batch_indices = [
+                    adapter_to_segment[idx] for idx in meta.adapter_indices.tolist()
+                ]
+                batch_indices = [
+                    idx if idx in rank_indices else -1 for idx in batch_indices
+                ]
+                batch_indices = torch.tensor(
+                    batch_indices, dtype=torch.int64, device=device
+                )
+
+            rank_data[rank] = RankSegments(
+                rank=rank,
+                tmp_shrink=tmp_shrink,
+                tmp_expand=tmp_expand,
+                lora_a_ptr=lora_a_ptr[indices],
+                lora_b_ptr=lora_b_ptr[indices],
+                segment_starts=segment_starts,
+                segment_ends=segment_ends,
+                indices=batch_indices,
+            )
+
+        return BatchLoraWeights(
+            lora_a=lora_a,
+            lora_b=lora_b,
+            adapter_index_configs=adapter_index_configs,
+            rank_data=rank_data,
+            use_sgmv=use_sgmv,
+        )
+
+
+def get_scaling_factor(
+    lora_alpha: int,
+    r: int,
+    uses_rslora: bool = False,
+) -> float:
+    """Computes the scaling factor for the lora weights."""
+    if uses_rslora:
+        return lora_alpha / (r**0.5)
+    return lora_alpha / r
+
+
+def _convert_lora(v: AdapterWeights) -> AdapterWeights:
+    if hasattr(v, "lora_weights"):
+        return v.lora_weights
+    return v
diff --git a/server/text_generation_server/adapters/weights.py b/server/text_generation_server/adapters/weights.py
new file mode 100644
index 00000000..da75dbcd
--- /dev/null
+++ b/server/text_generation_server/adapters/weights.py
@@ -0,0 +1,146 @@
+# Origin:   https://github.com/predibase/lorax
+# Path:     lorax/server/lorax_server/adapters/weights.py
+# License:  Apache License Version 2.0, January 2004
+
+from abc import ABC, abstractclassmethod
+from collections import defaultdict
+from dataclasses import dataclass
+from typing import Dict, List, Optional, Set, Type
+
+import torch
+
+
+@dataclass
+class AdapterBatchMetadata:
+    # [batch_size]
+    adapter_indices: torch.Tensor
+
+    # [num_adapters]
+    adapter_set: Set[int]
+
+    # [num_segments + 1]
+    adapter_segments: torch.Tensor
+
+    # [num_segments]
+    # maps from segment index to adapter index, i.e.:
+    # segment_indices[s] == adapter_indices[i]
+    segment_indices: List[int]
+
+
+class AdapterWeights(ABC):
+    @abstractclassmethod
+    def get_batch_types(cls) -> List[Type["BatchAdapterWeights"]]:
+        pass
+
+    @property
+    def speculative_tokens(self) -> int:
+        return 0
+
+
+class BatchAdapterWeights(ABC):
+    @abstractclassmethod
+    def has_adapter(self, adapter_index: int) -> bool:
+        pass
+
+    @abstractclassmethod
+    def load(
+        cls,
+        adapter_weights: Dict[int, AdapterWeights],
+        meta: "AdapterBatchMetadata",
+        prefill: bool,
+        prefill_head_indices: torch.Tensor,
+    ) -> Optional["BatchAdapterWeights"]:
+        pass
+
+
+class LayerAdapterWeights:
+    """Adapter weights that apply to a particular layer."""
+
+    def __init__(self):
+        self.adapter_weights: Dict[int, AdapterWeights] = {}
+
+    def add_adapter(self, adapter_idx: int, weights: AdapterWeights):
+        self.adapter_weights[adapter_idx] = weights
+
+    def remove_adapter(self, adapter_idx: int):
+        if adapter_idx not in self.adapter_weights:
+            return
+        del self.adapter_weights[adapter_idx]
+
+    def is_empty(self) -> bool:
+        return len(self.adapter_weights) == 0
+
+    def get_data(
+        self,
+        meta: AdapterBatchMetadata,
+        prefill: bool,
+        prefill_head_indices: Optional[torch.Tensor],
+    ) -> Dict[str, BatchAdapterWeights]:
+        # bucket adapters by batch class
+        adapter_batch_types: Dict[
+            Type[BatchAdapterWeights], Dict[int, AdapterWeights]
+        ] = defaultdict(dict)
+        for adapter_index, adapter_weights in self.adapter_weights.items():
+            for batch_type in adapter_weights.get_batch_types():
+                adapter_batch_types[batch_type][adapter_index] = adapter_weights
+
+        batch_data = {}
+        for batch_type, adapter_weights in adapter_batch_types.items():
+            batched_weights = batch_type.load(
+                adapter_weights, meta, prefill, prefill_head_indices
+            )
+            if batched_weights is not None:
+                batch_data = batched_weights
+        return batch_data
+
+
+@dataclass
+class AdapterBatchData:
+    meta: AdapterBatchMetadata
+
+    # layer type -> adapter type -> batch weight data
+    data: Dict[str, Dict[str, BatchAdapterWeights]]
+
+    prefill: bool
+
+    @staticmethod
+    def from_meta(
+        meta: AdapterBatchMetadata,
+        weights: Dict[str, LayerAdapterWeights],
+        prefill: bool,
+        prefill_head_indices: Optional[torch.Tensor],
+    ) -> "AdapterBatchData":
+        data = {}
+        for k, v in weights.items():
+            if v.is_empty():
+                continue
+            data[k] = v.get_data(
+                meta, prefill, prefill_head_indices if k == "lm_head" else None
+            )
+        return AdapterBatchData(meta=meta, data=data, prefill=prefill)
+
+    def ranks(self) -> Set[int]:
+        # TODO(travis): refactor to be less coupled to lora implementation
+        ranks = set()
+        for lora_data in self.data.values():
+            if lora_data is None:
+                continue
+
+            for rank_data in lora_data.rank_data.values():
+                ranks.add(rank_data.rank)
+
+        return ranks
+
+    def layer_names(self) -> Set[str]:
+        return set(self.data.keys())
+
+    def adapter_keys(self) -> Set[str]:
+        adapter_keys = set()
+        for layer_data in self.data.values():
+            adapter_keys.update(layer_data.keys())
+        return adapter_keys
+
+    @property
+    def max_rank(self) -> int:
+        ranks = self.ranks()
+        return max(ranks) if len(ranks) > 0 else 0
diff --git a/server/text_generation_server/cli.py b/server/text_generation_server/cli.py
index 74b87024..f9b4caa9 100644
--- a/server/text_generation_server/cli.py
+++ b/server/text_generation_server/cli.py
@@ -1,5 +1,3 @@
-# Copyright (C) 2024 Habana Labs, Ltd. an Intel Company.
-
 import os
 import psutil
 import signal
@@ -11,6 +9,7 @@ from loguru import logger
 from typing import Optional
 from enum import Enum
 from huggingface_hub import hf_hub_download
+from text_generation_server.utils.adapter import parse_lora_adapters
 
 
 app = typer.Typer()
@@ -18,10 +17,14 @@ app = typer.Typer()
 
 class Quantization(str, Enum):
     bitsandbytes = "bitsandbytes"
+    bitsandbytes_nf4 = "bitsandbytes-nf4"
+    bitsandbytes_fp4 = "bitsandbytes-fp4"
     gptq = "gptq"
     awq = "awq"
     eetq = "eetq"
+    exl2 = "exl2"
     fp8 = "fp8"
+    marlin = "marlin"
 
 
 class Dtype(str, Enum):
@@ -42,8 +45,13 @@ def serve(
     logger_level: str = "INFO",
     json_output: bool = False,
     otlp_endpoint: Optional[str] = None,
+    otlp_service_name: str = "text-generation-inference.server",
+    max_input_tokens: Optional[int] = None,
 ):
     if sharded:
+        # assert (
+        #     os.getenv("RANK", None) is not None
+        # ), "RANK must be set when sharded is True"
         assert (
             os.getenv("WORLD_SIZE", None) is not None
         ), "WORLD_SIZE must be set when sharded is True"
@@ -72,11 +80,35 @@ def serve(
 
     # Setup OpenTelemetry distributed tracing
     if otlp_endpoint is not None:
-        setup_tracing(shard=os.getenv("RANK", 0), otlp_endpoint=otlp_endpoint)
+        setup_tracing(otlp_service_name=otlp_service_name, otlp_endpoint=otlp_endpoint)
+
+    lora_adapters = parse_lora_adapters(os.getenv("LORA_ADAPTERS"))
+
+    # TODO: enable lora with cuda graphs. for now disable cuda graphs if lora is enabled
+    # and warn the user
+    if lora_adapters:
+        logger.warning("LoRA adapters enabled (experimental feature).")
+
+        if "CUDA_GRAPHS" in os.environ:
+            logger.warning(
+                "LoRA adapters incompatible with CUDA Graphs. Disabling CUDA Graphs."
+            )
+            global CUDA_GRAPHS
+            CUDA_GRAPHS = None
 
     # Downgrade enum into str for easier management later on
     quantize = None if quantize is None else quantize.value
     dtype = "bfloat16" if dtype is None else dtype.value
+    logger.info(f"quantize={quantize}")
+    if dtype is not None and quantize not in {
+        None,
+        "bitsandbytes",
+        "bitsandbytes-nf4",
+        "bitsandbytes-fp4",
+    }:
+        raise RuntimeError(
+            "Only 1 can be set between `dtype` and `quantize`, as they both decide how goes the final model."
+        )
 
     logger.info("CLI SHARDED = {} DTYPE = {}".format(sharded, dtype))
 
@@ -89,6 +121,7 @@ def serve(
         cmd = f"deepspeed --num_nodes 1 --num_gpus {num_shard} --no_local_rank {tgi_file}"
         cmd += f" --model_id {model_id} --revision {revision} --sharded {sharded}"
         cmd += f" --dtype {dtype} --trust_remote_code {trust_remote_code} --uds_path {uds_path}"
+        cmd += f" --quantize {quantize} --max_input_tokens {max_input_tokens}"
         if speculate is not None:
             cmd += f"--speculate {speculate}"
         logger.info("CLI server start deepspeed ={} ".format(cmd))
@@ -136,12 +169,15 @@ def serve(
     else:
         server.serve(
             model_id,
+            lora_adapters,
             revision,
             sharded,
+            quantize,
             speculate,
             dtype,
             trust_remote_code,
-            uds_path
+            uds_path,
+            max_input_tokens,
         )
 
 
@@ -154,6 +190,7 @@ def download_weights(
     logger_level: str = "INFO",
     json_output: bool = False,
     trust_remote_code: bool = False,
+    merge_lora: bool = False,
 ):
     # Remove default handler
     logger.remove()
@@ -184,18 +221,28 @@ def download_weights(
     ) is not None
 
     if not is_local_model:
-        try:
-            adapter_config_filename = hf_hub_download(
-                model_id, revision=revision, filename="adapter_config.json"
-            )
-            utils.download_and_unload_peft(
-                model_id, revision, trust_remote_code=trust_remote_code
-            )
-            is_local_model = True
-            utils.weight_files(model_id, revision, extension)
-            return
-        except (utils.LocalEntryNotFoundError, utils.EntryNotFoundError):
-            pass
+        # TODO: maybe reverse the default value of merge_lora?
+        # currently by default we don't merge the weights with the base model
+        if merge_lora:
+            try:
+                hf_hub_download(
+                    model_id, revision=revision, filename="adapter_config.json"
+                )
+                utils.download_and_unload_peft(
+                    model_id, revision, trust_remote_code=trust_remote_code
+                )
+                is_local_model = True
+                utils.weight_files(model_id, revision, extension)
+                return
+            except (utils.LocalEntryNotFoundError, utils.EntryNotFoundError):
+                pass
+        else:
+            try:
+                utils.peft.download_peft(
+                    model_id, revision, trust_remote_code=trust_remote_code
+                )
+            except Exception:
+                pass
 
         try:
             import json
@@ -298,9 +345,9 @@ def download_weights(
     if auto_convert:
         if not trust_remote_code:
             logger.warning(
-                f"🚨🚨BREAKING CHANGE in 2.0🚨🚨: Safetensors conversion is disabled without `--trust-remote-code` because "
-                f"Pickle files are unsafe and can essentially contain remote code execution!"
-                f"Please check for more information here: https://huggingface.co/docs/text-generation-inference/basic_tutorials/safety",
+                "🚨🚨BREAKING CHANGE in 2.0🚨🚨: Safetensors conversion is disabled without `--trust-remote-code` because "
+                "Pickle files are unsafe and can essentially contain remote code execution!"
+                "Please check for more information here: https://huggingface.co/docs/text-generation-inference/basic_tutorials/safety",
             )
 
         logger.warning(
@@ -309,22 +356,28 @@ def download_weights(
         )
 
         # Safetensors final filenames
-        local_st_files = [p.parent / f"{p.stem.lstrip('pytorch_')}.safetensors" for p in local_pt_files]
+        local_st_files = [
+            p.parent / f"{p.stem.lstrip('pytorch_')}.safetensors"
+            for p in local_pt_files
+        ]
         try:
             import transformers
-            from transformers import AutoConfig
+            import json
 
-            config = AutoConfig.from_pretrained(
-                model_id,
-                revision=revision,
-            )
-            architecture = config.architectures[0]
+            if is_local_model:
+                config_filename = os.path.join(model_id, "config.json")
+            else:
+                config_filename = hf_hub_download(
+                    model_id, revision=revision, filename="config.json"
+                )
+            with open(config_filename, "r") as f:
+                config = json.load(f)
+            architecture = config["architectures"][0]
 
             class_ = getattr(transformers, architecture)
 
             # Name for this varible depends on transformers version.
             discard_names = getattr(class_, "_tied_weights_keys", [])
-            discard_names.extend(getattr(class_, "_keys_to_ignore_on_load_missing", []))
 
         except Exception:
             discard_names = []
@@ -343,24 +396,29 @@ def quantize(
     upload_to_model_id: Optional[str] = None,
     percdamp: float = 0.01,
     act_order: bool = False,
+    groupsize: int = 128,
 ):
+    if revision is None:
+        revision = "main"
     download_weights(
         model_id=model_id,
         revision=revision,
         logger_level=logger_level,
         json_output=json_output,
     )
-    from text_generation_server.utils.gptq.quantize import quantize
+    from text_generation_server.layers.gptq.quantize import quantize
 
     quantize(
         model_id=model_id,
         bits=4,
-        groupsize=128,
+        groupsize=groupsize,
         output_dir=output_dir,
+        revision=revision,
         trust_remote_code=trust_remote_code,
         upload_to_model_id=upload_to_model_id,
         percdamp=percdamp,
         act_order=act_order,
+        sym=True,
     )
 
 
diff --git a/server/text_generation_server/layers/__init__.py b/server/text_generation_server/layers/__init__.py
index c29dd092..0000ca91 100644
--- a/server/text_generation_server/layers/__init__.py
+++ b/server/text_generation_server/layers/__init__.py
@@ -12,3 +12,23 @@ from text_generation_server.layers.speculative import SpeculativeHead
 # Just to add the `load` methods.
 from text_generation_server.layers.layernorm import load_layer_norm
 from text_generation_server.layers.conv import load_conv2d
+
+from text_generation_server.layers.lora import (
+    LoraLinear,
+    TensorParallelMultiAdapterLinear,
+    TensorParallelAdapterRowLinear,
+)
+
+__all__ = [
+    "get_linear",
+    "FastLinear",
+    "TensorParallelColumnLinear",
+    "TensorParallelRowLinear",
+    "TensorParallelEmbedding",
+    "SpeculativeHead",
+    "LoraLinear",
+    "TensorParallelMultiAdapterLinear",
+    "TensorParallelAdapterRowLinear",
+    "load_layer_norm",
+    "load_conv2d",
+]
diff --git a/server/text_generation_server/layers/attention/__init__.py b/server/text_generation_server/layers/attention/__init__.py
new file mode 100644
index 00000000..4f2b9807
--- /dev/null
+++ b/server/text_generation_server/layers/attention/__init__.py
@@ -0,0 +1,43 @@
+from text_generation_server.utils.import_utils import SYSTEM
+import os
+
+from .common import Seqlen
+
+if os.getenv("USE_FLASH_ATTENTION", "").lower() == "false":
+    raise ImportError("`USE_FLASH_ATTENTION` is false.")
+if SYSTEM == "cuda":
+    from .cuda import (
+        attention,
+        paged_attention,
+        reshape_and_cache,
+        SUPPORTS_WINDOWING,
+        PREFILL_IN_KV_CACHE,
+    )
+elif SYSTEM == "rocm":
+    from .rocm import (
+        attention,
+        paged_attention,
+        reshape_and_cache,
+        PREFILL_IN_KV_CACHE,
+        SUPPORTS_WINDOWING,
+    )
+elif SYSTEM == "ipex":
+    from .ipex import (
+        attention,
+        paged_attention,
+        reshape_and_cache,
+        PREFILL_IN_KV_CACHE,
+        SUPPORTS_WINDOWING,
+    )
+else:
+    raise ImportError(f"System {SYSTEM} doesn't support flash/paged attention")
+
+
+__all__ = [
+    "attention",
+    "paged_attention",
+    "reshape_and_cache",
+    "PREFILL_IN_KV_CACHE",
+    "SUPPORTS_WINDOWING",
+    "Seqlen",
+]
diff --git a/server/text_generation_server/layers/attention/common.py b/server/text_generation_server/layers/attention/common.py
new file mode 100644
index 00000000..d6e512c0
--- /dev/null
+++ b/server/text_generation_server/layers/attention/common.py
@@ -0,0 +1,72 @@
+from dataclasses import dataclass
+from text_generation_server.utils.import_utils import SYSTEM
+from text_generation_server.models.globals import ATTENTION
+import torch
+from typing import Optional
+
+
+if ATTENTION in {"flashinfer", "flashdecoding"}:
+
+    @dataclass
+    class Seqlen:
+        input_lengths: torch.Tensor
+        prefix_lengths: torch.Tensor
+        cu_seqlen_q: Optional[torch.Tensor]
+        cu_seqlen_k: Optional[torch.Tensor]
+        max_q: int
+        max_k: int
+
+        def __init__(
+            self,
+            input_lengths,
+            prefix_lengths,
+            cu_seqlen_q=None,
+            max_q=None,
+            max_k=None,
+        ):
+            self.input_lengths = input_lengths
+            self.prefix_lengths = prefix_lengths
+            device = self.input_lengths.device
+            shape = self.input_lengths.shape
+            if cu_seqlen_q is None:
+                cu_seqlen_q = torch.arange(
+                    shape[0] + 1,
+                    device=device,
+                    dtype=torch.int32,
+                )
+                max_q = 1
+            else:
+                assert max_q is not None
+            assert max_k is not None
+            cu_seqlen_k = torch.zeros(shape[-1] + 1, device=device, dtype=torch.int32)
+
+            # cuda graphs don't like this and this is necessary to clamp within mistral
+            # Although FA2 might not want the clamping
+            # cu_seqlen_k[0] = 0
+            total = self.input_lengths + self.prefix_lengths
+            torch.cumsum(total, -1, out=cu_seqlen_k[1:])
+
+            self.cu_seqlen_q = cu_seqlen_q
+            self.cu_seqlen_k = cu_seqlen_k
+            self.max_q = max_q
+            self.max_k = max_k
+
+        def clamp(self, max):
+            # Flash decoding doesn't need to clamp
+            return self
+
+else:
+
+    @dataclass
+    class Seqlen:
+        input_lengths: torch.Tensor
+        prefix_lengths: torch.Tensor
+        cu_seqlen_q: torch.Tensor
+        max_q: int
+        max_k: int
+
+        def clamp(self, max):
+            if SYSTEM == "rocm":
+                return self
+            raise NotImplementedError("Not implemented seqlen for paged")
+            return Seqlen(torch.clamp(self.input_lengths, max=max))
diff --git a/server/text_generation_server/layers/attention/cuda.py b/server/text_generation_server/layers/attention/cuda.py
new file mode 100644
index 00000000..51af928d
--- /dev/null
+++ b/server/text_generation_server/layers/attention/cuda.py
@@ -0,0 +1,357 @@
+import torch
+from text_generation_server.utils.import_utils import SYSTEM
+from text_generation_server.models.globals import (
+    ATTENTION,
+    BLOCK_SIZE,
+)
+from text_generation_server.layers.attention import Seqlen
+from typing import Optional
+
+major, minor = torch.cuda.get_device_capability()
+is_sm75 = major == 7 and minor == 5
+_PARTITION_SIZE = 512
+
+try:
+    from vllm._C import cache_ops
+except Exception as e:
+    raise ImportError(
+        f"Could not import vllm paged attention. Make sure your installation is correct. Complete error: {e}"
+    )
+
+
+def reshape_and_cache(
+    key: torch.Tensor,
+    value: torch.Tensor,
+    key_cache: torch.Tensor,
+    value_cache: torch.Tensor,
+    slots: torch.Tensor,
+):
+    if ATTENTION in {"flashdecoding", "flashinfer"}:
+        shape = key_cache.shape
+        key_cache.view(-1, shape[-2], shape[-1])[slots] = key
+        value_cache.view(-1, shape[-2], shape[-1])[slots] = value
+    else:
+        cache_ops.reshape_and_cache(
+            key, value, key_cache, value_cache, slots, "auto", 1.0
+        )
+
+
+def paged_attention(
+    query: torch.Tensor,
+    key_cache: torch.Tensor,
+    value_cache: torch.Tensor,
+    kv_head_mapping: torch.Tensor,
+    softmax_scale: float,
+    block_tables: torch.Tensor,
+    seqlen: Seqlen,
+    max_s: int,
+    softcap: Optional[float] = None,
+):
+    # Adapted from: https://github.com/vllm-project/vllm/blob/f8a1e39fae05ca610be8d5a78be9d40f5274e5fc/vllm/model_executor/layers/attention.py
+    # Copyright 2023 The vLLM team. All rights
+    # reserved.
+    #
+    # Licensed under the Apache License, Version 2.0 (the "License");
+    # you may not use this file except in compliance with the License.
+    # You may obtain a copy of the License at
+    #
+    #     http://www.apache.org/licenses/LICENSE-2.0
+    #
+    # Unless required by applicable law or agreed to in writing, software
+    # distributed under the License is distributed on an "AS IS" BASIS,
+    # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    # See the License for the specific language governing permissions and
+    # limitations under the License.
+    #
+
+    # value_cache => [num_blocks, num_heads, head_size, block_size]
+    # block_size = value_cache.shape[3]
+    block_size = BLOCK_SIZE
+    num_seqs, num_heads, head_size = query.shape
+    max_num_partitions = (max_s + _PARTITION_SIZE - 1) // _PARTITION_SIZE
+
+    # NOTE(woosuk): We use a simple heuristic to decide whether to use
+    # PagedAttention V1 or V2. If the number of partitions is 1, we use
+    # V1 to avoid the overhead of reduction. Also, if the number of
+    # sequences or heads is large, we use V1 since there is enough work
+    # to parallelize.
+    if ATTENTION == "flashinfer":
+        from text_generation_server.layers.attention.flashinfer import decode_state
+
+        return decode_state.get().forward(
+            query.contiguous(),
+            paged_kv_cache=(key_cache, value_cache),
+            logits_soft_cap=softcap,
+            sm_scale=softmax_scale,
+        )
+    elif ATTENTION == "flashdecoding":
+        max_q = 1
+        max_k = max_s
+        import flash_attn_2_cuda
+
+        # TODO fixme when flash contains the fix.
+        # Number of splits is not correctly handled
+        # by the current path
+        # https://github.com/Dao-AILab/flash-attention/blob/320fb59487658f033f56711efd3d61b7c7a6f8f3/csrc/flash_attn/flash_api.cpp#L577
+        # This fails becuase we're using causal, therefore window_right is set to 0 and the split logic is never applied.
+        if softcap is None:
+            softcap = 0.0
+        out = flash_attn_2_cuda.varlen_fwd(
+            query,
+            key_cache,
+            value_cache,
+            None,
+            seqlen.cu_seqlen_q,
+            seqlen.cu_seqlen_k,
+            None,  # pad_k
+            None,
+            block_tables,
+            None,
+            max_q,
+            max_k,
+            0.0,  # dropout
+            softmax_scale,
+            False,  # zero_tensors
+            True,  # causal
+            -1,  # Window_left
+            -1,  # Window right
+            softcap,
+            False,  # return softmax
+            None,  # generator
+        )
+        return out[0]
+    else:
+        if softcap is not None:
+            raise RuntimeError("Paged attention doesn't support softcapping")
+        input_lengths = seqlen.input_lengths
+        from vllm._C import ops
+
+        out = torch.empty_like(query)
+
+        use_v1 = max_s <= 8192 and (
+            max_num_partitions == 1 or num_seqs * num_heads > 512
+        )
+        if use_v1:
+            ops.paged_attention_v1(
+                out,
+                query,
+                key_cache,
+                value_cache,
+                kv_head_mapping,
+                softmax_scale,
+                block_tables,
+                input_lengths,
+                block_size,
+                max_s,
+                None,
+                "auto",
+                1.0,
+            )
+        else:
+            # Run PagedAttention V2.
+            assert _PARTITION_SIZE % block_size == 0
+            tmp_output = torch.empty(
+                size=(num_seqs, num_heads, max_num_partitions, head_size),
+                dtype=out.dtype,
+                device=out.device,
+            )
+            exp_sums = torch.empty(
+                size=(num_seqs, num_heads, max_num_partitions),
+                dtype=torch.float32,
+                device=out.device,
+            )
+            max_logits = torch.empty_like(exp_sums)
+
+            ops.paged_attention_v2(
+                out,
+                exp_sums,
+                max_logits,
+                tmp_output,
+                query,
+                key_cache,
+                value_cache,
+                kv_head_mapping,
+                softmax_scale,
+                block_tables,
+                input_lengths,
+                block_size,
+                max_s,
+                None,
+                "auto",
+                1.0,
+            )
+    return out
+
+
+try:
+    is_ampere_or_newer = major >= 8 and minor >= 0
+    if not is_ampere_or_newer:
+        raise ImportError("FlashAttention only supports Ampere GPUs or newer.")
+
+    import flash_attn_2_cuda
+
+    V2 = True
+except ImportError:
+    try:
+        import flash_attn_cuda
+
+        V2 = False
+    except ImportError as e:
+        if major >= 8:
+            architecture_suffix = f"-{SYSTEM}"
+            raise ImportError(
+                "Flash Attention V2 is not installed.\n"
+                "Use the official Docker image (ghcr.io/huggingface/text-generation-inference:latest) "
+                f"or install flash attention v2 with `cd server && make install install-flash-attention-v2{architecture_suffix}`"
+            )
+        elif is_sm75:
+            raise ImportError(
+                "Flash Attention is not installed.\n"
+                "Use the official Docker image (ghcr.io/huggingface/text-generation-inference:latest) "
+                "or install flash attention with `cd server && make install install-flash-attention`"
+            ) from e
+        else:
+            raise ImportError(
+                f"GPU with CUDA capability {major} {minor} is not supported"
+            ) from e
+
+
+SUPPORTS_WINDOWING = V2
+
+if ATTENTION == "flashinfer":
+
+    def attention(
+        q: torch.Tensor,
+        key_cache: torch.Tensor,
+        value_cache: torch.Tensor,
+        seqlen: Seqlen,
+        block_tables: torch.Tensor,
+        softmax_scale,
+        window_size_left=-1,
+        causal=True,
+        softcap=0.0,
+    ):
+        from text_generation_server.layers.attention.flashinfer import (
+            prefill_with_paged_kv_state,
+        )
+
+        return prefill_with_paged_kv_state.get().forward(
+            q.contiguous(),
+            causal=causal,
+            paged_kv_cache=(key_cache, value_cache),
+            logits_soft_cap=softcap,
+            sm_scale=softmax_scale,
+            window_left=window_size_left,
+        )
+
+elif V2:
+
+    def attention(
+        q,
+        key_cache: torch.Tensor,
+        value_cache: torch.Tensor,
+        seqlen: Seqlen,
+        block_tables: torch.Tensor,
+        softmax_scale,
+        window_size_left=-1,
+        causal=True,
+        softcap=0.0,
+    ):
+        out = torch.empty_like(q)
+        if window_size_left <= 0 and window_size_left != -1:
+            raise ValueError("`window_size_left` must be > 0 or -1")
+        return flash_attn_2_cuda.varlen_fwd(
+            q,
+            key_cache,
+            value_cache,
+            out,
+            seqlen.cu_seqlen_q,
+            seqlen.cu_seqlen_k,
+            None,
+            None,
+            block_tables,
+            None,
+            seqlen.max_q,
+            seqlen.max_k,
+            0.0,
+            softmax_scale,
+            False,
+            causal,
+            window_size_left,
+            0,
+            softcap,
+            False,
+            None,
+        )[0]
+
+else:
+
+    def attention(
+        q: torch.Tensor,
+        k: torch.Tensor,
+        v: torch.Tensor,
+        seqlen: Seqlen,
+        block_tables: torch.Tensor,
+        softmax_scale: float,
+        window_size_left: int = -1,
+        causal: bool = True,
+        softcap=None,
+    ):
+        if window_size_left != -1:
+            raise NotImplementedError(
+                "window_size_left is only available with flash attn v2"
+            )
+        if softcap is not None:
+            raise NotImplementedError("softcap is only available with flash attn v2")
+
+        # Flash attention v1 requires q, k and v to have the same number of heads
+        if k.shape[1] != q.shape[1]:
+            # MQA expand
+            if k.shape[1] == 1:
+                k = k.expand(-1, q.shape[1], -1)
+            # Grouped attention reshape
+            else:
+                original_shape = k.shape
+                k = (
+                    k.unsqueeze(2)
+                    .expand(-1, -1, q.shape[1] // k.shape[1], -1)
+                    .reshape(original_shape[0], -1, original_shape[2])
+                )
+        if v.shape[1] != q.shape[1]:
+            # MQA expand
+            if v.shape[1] == 1:
+                v = v.expand(-1, q.shape[1], -1)
+            # Grouped attention reshape
+            else:
+                original_shape = v.shape
+                v = (
+                    v.unsqueeze(2)
+                    .expand(-1, -1, q.shape[1] // v.shape[1], -1)
+                    .reshape(original_shape[0], -1, original_shape[2])
+                )
+
+        out = torch.empty_like(q)
+        flash_attn_cuda.fwd(
+            q,
+            k,
+            v,
+            out,
+            seqlen.cu_seqlen_q,
+            seqlen.cu_seqlen_q,
+            seqlen.max_q,
+            seqlen.max_k,
+            0.0,
+            softmax_scale,
+            False,
+            causal,
+            False,
+            0,
+            None,
+        )
+        return out
+
+
+# Prefill in the cache with every kind of attention, unless we
+# have a configuration that requires flash-attention v1, which
+# does not support block tables.
+PREFILL_IN_KV_CACHE = ATTENTION != "paged" or V2
diff --git a/server/text_generation_server/utils/flash_attn_triton.py b/server/text_generation_server/layers/attention/flash_attn_triton.py
similarity index 99%
rename from server/text_generation_server/utils/flash_attn_triton.py
rename to server/text_generation_server/layers/attention/flash_attn_triton.py
index 3fe32231..3a6f9a73 100644
--- a/server/text_generation_server/utils/flash_attn_triton.py
+++ b/server/text_generation_server/layers/attention/flash_attn_triton.py
@@ -747,11 +747,8 @@ class _attention(torch.autograd.Function):
         padded_d_model = 1 << (head_size - 1).bit_length()
         padded_d_model = max(padded_d_model, 16)
 
-        grid = lambda META: (
-            triton.cdiv(max_seqlens_q, META["BLOCK_M"]),
-            nheads_q,
-            batch,
-        )
+        def grid(META):
+            return triton.cdiv(max_seqlens_q, META["BLOCK_M"]), nheads_q, batch
 
         encoded_softmax = None
 
diff --git a/server/text_generation_server/layers/attention/flashinfer.py b/server/text_generation_server/layers/attention/flashinfer.py
new file mode 100644
index 00000000..d603c6f5
--- /dev/null
+++ b/server/text_generation_server/layers/attention/flashinfer.py
@@ -0,0 +1,251 @@
+from typing import Optional
+from contextvars import ContextVar
+from contextlib import contextmanager
+
+import flashinfer
+import torch
+
+prefill_state: ContextVar[flashinfer.BatchPrefillWithRaggedKVCacheWrapper] = ContextVar(
+    "prefill_state"
+)
+
+prefill_with_paged_kv_state: ContextVar[
+    flashinfer.BatchPrefillWithPagedKVCacheWrapper
+] = ContextVar("prefill_with_paged_kv_state")
+
+decode_state: ContextVar[flashinfer.BatchDecodeWithPagedKVCacheWrapper] = ContextVar(
+    "decode_state"
+)
+
+workspace: Optional[torch.Tensor] = None
+
+
+def get_workspace(device):
+    """Get shared flashinfer workspace."""
+    global workspace
+    if workspace is None:
+        workspace = torch.empty(128 * 1024 * 1024, dtype=torch.uint8, device=device)
+    return workspace
+
+
+def create_prefill_with_paged_kv_state(
+    *,
+    device: torch.device,
+):
+    """Create a prefill state that uses the KV cache."""
+    workspace_buffer = get_workspace(device)
+    return flashinfer.BatchPrefillWithPagedKVCacheWrapper(
+        workspace_buffer, kv_layout="NHD", use_cuda_graph=False
+    )
+
+
+@contextmanager
+def use_prefill_with_paged_kv_state(
+    *,
+    state: flashinfer.BatchPrefillWithPagedKVCacheWrapper,
+    block_tables: torch.Tensor,
+    cu_seqlens: torch.Tensor,
+    input_lengths: torch.Tensor,
+    num_heads: int,
+    num_kv_heads: int,
+    head_size: int,
+    page_size: int,
+    dtype: torch.dtype,
+    window_left: int,
+):
+    """
+    Context manager to set the active flashinfer prefill state to the given
+    `state` and parameters. This state will be used by all calls to the
+    `attention` function while the context manager is active.
+    """
+
+    indptr = torch.zeros(
+        input_lengths.shape[0] + 1, device=input_lengths.device, dtype=torch.int32
+    )
+    # Round up to page size and then calculate the cumulative sum to get
+    # the indices into the block table.
+    torch.add(input_lengths, page_size - 1, out=indptr[1:])
+    indptr[1:].div_(page_size, rounding_mode="floor")
+    indptr[1:].cumsum_(-1)
+
+    # Get the lengths of the last page in a block.
+    if page_size == 1:
+        last_page_len = torch.ones(
+            input_lengths.shape[0], dtype=torch.int32, device=input_lengths.device
+        )
+    else:
+        last_page_len = torch.empty(
+            input_lengths.shape[0], dtype=torch.int32, device=input_lengths.device
+        )
+        torch.sub(input_lengths, 1, out=last_page_len)
+        last_page_len.remainder_(page_size)
+        last_page_len += 1
+
+    token = prefill_with_paged_kv_state.set(state)
+    try:
+        state.begin_forward(
+            qo_indptr=cu_seqlens,
+            paged_kv_indptr=indptr,
+            paged_kv_indices=block_tables,
+            paged_kv_last_page_len=last_page_len,
+            num_qo_heads=num_heads,
+            num_kv_heads=num_kv_heads,
+            head_dim=head_size,
+            q_data_type=dtype,
+            page_size=page_size,
+            window_left=window_left,
+        )
+        yield
+    finally:
+        state.end_forward()
+        if token is not None:
+            prefill_with_paged_kv_state.reset(token)
+
+
+def create_prefill_state(
+    *,
+    device: torch.device,
+):
+    """Create a prefill state."""
+    workspace_buffer = get_workspace(device)
+    return flashinfer.BatchPrefillWithRaggedKVCacheWrapper(
+        workspace_buffer, kv_layout="NHD", use_cuda_graph=False
+    )
+
+
+@contextmanager
+def use_prefill_state(
+    *,
+    state: flashinfer.BatchPrefillWithRaggedKVCacheWrapper,
+    cu_seqlens: torch.Tensor,
+    num_heads: int,
+    num_kv_heads: int,
+    head_size: int,
+    dtype: torch.dtype,
+    window_left: int,
+):
+    """
+    Context manager to set the active flashinfer prefill state to the given
+    `state` and parameters. This state will be used by all calls to the
+    `attention` function while the context manager is active.
+    """
+
+    token = prefill_state.set(state)
+    try:
+        state.begin_forward(
+            qo_indptr=cu_seqlens,
+            kv_indptr=cu_seqlens,
+            num_qo_heads=num_heads,
+            num_kv_heads=num_kv_heads,
+            head_dim=head_size,
+            q_data_type=dtype,
+            window_left=window_left,
+        )
+        yield
+    finally:
+        state.end_forward()
+        if token is not None:
+            prefill_state.reset(token)
+
+
+def create_decode_state(
+    *,
+    device: torch.device,
+    num_heads: int,
+    num_kv_heads: int,
+):
+    """Create a decode state."""
+    workspace_buffer = get_workspace(device)
+    num_groups = num_heads // num_kv_heads
+    return flashinfer.BatchDecodeWithPagedKVCacheWrapper(
+        workspace_buffer,
+        kv_layout="NHD",
+        use_cuda_graph=False,
+        # Taken from https://github.com/flashinfer-ai/flashinfer/blob/33ef95700981ba70f4cab63b8931e562bc795b21/python/flashinfer/decode.py#L57-L60
+        use_tensor_cores=num_groups not in [1, 2, 4, 8],
+    )
+
+
+def create_decode_state_cuda_graphs(
+    *,
+    device: torch.device,
+    block_tables: torch.Tensor,
+    block_tables_ptr: torch.Tensor,
+    last_page_len: torch.Tensor,
+    num_heads: int,
+    num_kv_heads: int,
+):
+    """
+    Create a decode state for use with CUDA Graphs. `block_tables`,
+    `block_tables_ptr`, and `last_page_len` are used in CUDA Graphs and are
+    therefore stored as part of the state.
+    """
+    workspace_buffer = get_workspace(device)
+    num_groups = num_heads // num_kv_heads
+    return flashinfer.BatchDecodeWithPagedKVCacheWrapper(
+        workspace_buffer,
+        kv_layout="NHD",
+        use_cuda_graph=True,
+        paged_kv_indices_buffer=block_tables,
+        paged_kv_indptr_buffer=block_tables_ptr,
+        paged_kv_last_page_len_buffer=last_page_len,
+        # Taken from https://github.com/flashinfer-ai/flashinfer/blob/33ef95700981ba70f4cab63b8931e562bc795b21/python/flashinfer/decode.py#L57-L60
+        use_tensor_cores=num_groups not in [1, 2, 4, 8],
+    )
+
+
+@contextmanager
+def use_decode_state(
+    *,
+    state: flashinfer.BatchDecodeWithPagedKVCacheWrapper,
+    input_lengths: torch.Tensor,
+    block_tables: torch.Tensor,
+    num_heads: int,
+    num_kv_heads: int,
+    head_size: int,
+    page_size: int,
+    dtype: torch.dtype,
+    window_left: int,
+):
+    """
+    Context manager to set the active flashinfer decoding state to the given
+    `state` and parameters. This state will be used by all calls to the
+    `paged_attention` function while the context manager is active.
+    """
+    indptr = torch.zeros(
+        input_lengths.shape[0] + 1, device=input_lengths.device, dtype=torch.int32
+    )
+    # Round up to page size and then calculate the cumulative sum to get
+    # the indices into the block table.
+    torch.add(input_lengths, page_size - 1, out=indptr[1:])
+    indptr[1:].div_(page_size, rounding_mode="floor")
+    indptr[1:].cumsum_(-1)
+
+    # Get the lengths of the last page in a block.
+    last_page_len = torch.empty(
+        input_lengths.shape[0], dtype=torch.int32, device=input_lengths.device
+    )
+    torch.sub(input_lengths, 1, out=last_page_len)
+    last_page_len.remainder_(page_size)
+    last_page_len += 1
+
+    token = decode_state.set(state)
+
+    try:
+        state.begin_forward(
+            indptr=indptr,
+            indices=block_tables,
+            last_page_len=last_page_len,
+            num_qo_heads=num_heads,
+            num_kv_heads=num_kv_heads,
+            head_dim=head_size,
+            page_size=page_size,
+            data_type=dtype,
+            q_data_type=dtype,
+            window_left=window_left,
+        )
+        yield
+    finally:
+        state.end_forward()
+        if token is not None:
+            decode_state.reset(token)
diff --git a/server/text_generation_server/layers/attention/ipex.py b/server/text_generation_server/layers/attention/ipex.py
new file mode 100644
index 00000000..657c90af
--- /dev/null
+++ b/server/text_generation_server/layers/attention/ipex.py
@@ -0,0 +1,82 @@
+import intel_extension_for_pytorch as ipex
+import torch
+from text_generation_server.models.flash_causal_lm import BLOCK_SIZE
+from text_generation_server.layers.attention import Seqlen
+from typing import Optional
+
+SUPPORTS_WINDOWING = False
+PREFILL_IN_KV_CACHE = False
+
+
+def attention(
+    q: torch.Tensor,
+    key_cache: torch.Tensor,
+    value_cache: torch.Tensor,
+    seqlen: Seqlen,
+    block_tables: torch.Tensor,
+    softmax_scale,
+    window_size_left=-1,
+    causal=True,
+    softcap: Optional[float] = None,
+):
+    out = torch.empty_like(q)
+
+    # We do not need to check window_size_left (not supported) here, so it is already checked ahead of time at model load.
+    ipex.llm.functional.varlen_attention(
+        q.contiguous() if q.device.type == "xpu" else q,
+        key_cache.contiguous() if key_cache.device.type == "xpu" else key_cache,
+        value_cache.contiguous() if value_cache.device.type == "xpu" else value_cache,
+        out,
+        seqlen.cu_seqlen_q,
+        seqlen.cu_seqlen_q,
+        seqlen.max_q,
+        seqlen.max_q,
+        0.0,
+        softmax_scale,
+        False,
+        causal,
+        False,
+        None,
+    )
+
+    return out
+
+
+def reshape_and_cache(
+    key: torch.Tensor,
+    value: torch.Tensor,
+    key_cache: torch.Tensor,
+    value_cache: torch.Tensor,
+    slots: torch.Tensor,
+):
+    ipex.llm.modules.PagedAttention.reshape_and_cache(
+        key, value, key_cache, value_cache, slots
+    )
+
+
+def paged_attention(
+    query: torch.Tensor,
+    key_cache: torch.Tensor,
+    value_cache: torch.Tensor,
+    kv_head_mapping: torch.Tensor,
+    softmax_scale: float,
+    block_tables: torch.Tensor,
+    seqlen: Seqlen,
+    max_s: int,
+    softcap: Optional[float] = None,
+):
+    out = torch.empty_like(query)
+    ipex.llm.modules.PagedAttention.single_query_cached_kv_attention(
+        out,
+        query,
+        key_cache,
+        value_cache,
+        kv_head_mapping,
+        softmax_scale,
+        block_tables,
+        seqlen.input_lengths,
+        BLOCK_SIZE,
+        max_s,
+        None,
+    )
+    return out
diff --git a/server/text_generation_server/layers/attention/rocm.py b/server/text_generation_server/layers/attention/rocm.py
new file mode 100644
index 00000000..646a763d
--- /dev/null
+++ b/server/text_generation_server/layers/attention/rocm.py
@@ -0,0 +1,308 @@
+import os
+from typing import Optional
+import torch
+from text_generation_server.utils.import_utils import SYSTEM
+from text_generation_server.models.globals import ATTENTION
+from text_generation_server.layers.attention import Seqlen
+from text_generation_server.utils.log import log_master
+from loguru import logger
+
+major, minor = torch.cuda.get_device_capability()
+is_sm75 = major == 7 and minor == 5
+
+_PARTITION_SIZE_V1V2 = 512
+_PARTITION_SIZE_CUSTOM = 256
+
+use_triton = os.getenv("ROCM_USE_FLASH_ATTN_V2_TRITON", "").lower() in {"true", "1"}
+ENGINE = "triton" if use_triton else "ck"
+
+PREFILL_IN_KV_CACHE = False
+
+use_rocm_custom_paged_attn = os.getenv("ROCM_USE_CUSTOM_PAGED_ATTN", "1") != "0"
+try:
+    if use_rocm_custom_paged_attn:
+        from vllm._custom_C import paged_attention_custom
+except ImportError as e:
+    log_master(
+        logger.info,
+        f"Custom Paged Attention not available. Complete error: {e}",
+    )
+    use_rocm_custom_paged_attn = False
+
+try:
+    import vllm._custom_ops as ops
+except Exception as e:
+    raise ImportError(
+        f"Could not import vllm paged attention. Make sure your installation is correct. Complete error: {e}"
+    )
+
+
+def reshape_and_cache(
+    key: torch.Tensor,
+    value: torch.Tensor,
+    key_cache: torch.Tensor,
+    value_cache: torch.Tensor,
+    slots: torch.Tensor,
+):
+    if ATTENTION == "flashdecoding":
+        shape = key_cache.shape
+        key_cache.view(-1, shape[-2], shape[-1])[slots] = key
+        value_cache.view(-1, shape[-2], shape[-1])[slots] = value
+    else:
+        ops.reshape_and_cache(key, value, key_cache, value_cache, slots, "auto", 1.0)
+
+
+def paged_attention(
+    query: torch.Tensor,
+    key_cache: torch.Tensor,
+    value_cache: torch.Tensor,
+    kv_head_mapping: torch.Tensor,
+    softmax_scale: float,
+    block_tables: torch.Tensor,
+    seqlen: Seqlen,
+    max_s: int,
+    softcap: Optional[float] = None,
+):
+    # Adapted from: https://github.com/vllm-project/vllm/blob/f8a1e39fae05ca610be8d5a78be9d40f5274e5fc/vllm/model_executor/layers/attention.py
+    # Copyright 2023 The vLLM team. All rights
+    # reserved.
+    #
+    # Licensed under the Apache License, Version 2.0 (the "License");
+    # you may not use this file except in compliance with the License.
+    # You may obtain a copy of the License at
+    #
+    #     http://www.apache.org/licenses/LICENSE-2.0
+    #
+    # Unless required by applicable law or agreed to in writing, software
+    # distributed under the License is distributed on an "AS IS" BASIS,
+    # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    # See the License for the specific language governing permissions and
+    # limitations under the License.
+    #
+
+    if softcap is not None:
+        raise RuntimeError("Paged attention doesn't support softcapping")
+
+    # value_cache => [num_blocks, num_heads, head_size, block_size]
+    block_size = value_cache.shape[3]
+    num_seqs, num_heads, head_size = query.shape
+
+    num_kv_heads = key_cache.shape[1]
+    gqa_ratio = num_heads // num_kv_heads
+    use_custom = (
+        use_rocm_custom_paged_attn
+        and (query.dtype == torch.half or query.dtype == torch.bfloat16)
+        and (head_size == 128 or head_size == 64)
+        and (block_size == 16 or block_size == 32)
+        and (gqa_ratio >= 1 and gqa_ratio <= 16)
+        and max_s <= 32768
+    )
+
+    if not use_custom:
+        _PARTITION_SIZE = _PARTITION_SIZE_V1V2
+    else:
+        _PARTITION_SIZE = _PARTITION_SIZE_CUSTOM
+
+    max_num_partitions = (max_s + _PARTITION_SIZE - 1) // _PARTITION_SIZE
+    input_lengths = seqlen.input_lengths
+
+    out = torch.empty_like(query)
+
+    # NOTE(woosuk): We use a simple heuristic to decide whether to use
+    # PagedAttention V1 or V2. If the number of partitions is 1, we use
+    # V1 to avoid the overhead of reduction. Also, if the number of
+    # sequences or heads is large, we use V1 since there is enough work
+    # to parallelize.
+    import vllm._custom_ops as ops
+
+    use_v1 = (
+        max_s <= 8192
+        and (max_num_partitions == 1 or num_seqs * num_heads > 512)
+        and not use_custom
+    )
+    if use_v1:
+        ops.paged_attention_v1(
+            out,
+            query,
+            key_cache,
+            value_cache,
+            kv_head_mapping,
+            softmax_scale,
+            block_tables,
+            input_lengths,
+            block_size,
+            max_s,
+            None,
+            "auto",
+            1.0,
+        )
+    else:
+        # Run PagedAttention V2.
+        assert _PARTITION_SIZE % block_size == 0
+        tmp_output = torch.empty(
+            size=(num_seqs, num_heads, max_num_partitions, head_size),
+            dtype=out.dtype,
+            device=out.device,
+        )
+        exp_sums = torch.empty(
+            size=(num_seqs, num_heads, max_num_partitions),
+            dtype=torch.float32,
+            device=out.device,
+        )
+        max_logits = torch.empty_like(exp_sums)
+
+        if not use_custom:
+            ops.paged_attention_v2(
+                out,
+                exp_sums,
+                max_logits,
+                tmp_output,
+                query,
+                key_cache,
+                value_cache,
+                kv_head_mapping,
+                softmax_scale,
+                block_tables,
+                input_lengths,
+                block_size,
+                max_s,
+                None,
+                "auto",
+                1.0,
+            )
+        else:
+            paged_attention_custom(
+                out,
+                exp_sums,
+                max_logits,
+                tmp_output,
+                query,
+                key_cache,
+                value_cache,
+                num_kv_heads,
+                softmax_scale,
+                block_tables,
+                input_lengths,
+                block_size,
+                max_s,
+                None,
+                "auto",
+            )
+
+    return out
+
+
+if ENGINE != "triton":
+    try:
+        import flash_attn_2_cuda
+
+        log_master(
+            logger.info,
+            "ROCm: using Flash Attention 2 Composable Kernel implementation.",
+        )
+    except ImportError as e:
+        if major >= 8:
+            architecture_suffix = f"-{SYSTEM}"
+            raise ImportError(
+                "Flash Attention V2 is not installed.\n"
+                "Use the official Docker image (ghcr.io/huggingface/text-generation-inference:latest) "
+                f"or install flash attention v2 with `cd server && make install install-flash-attention-v2{architecture_suffix}`"
+            )
+        elif is_sm75:
+            raise ImportError(
+                "Flash Attention is not installed.\n"
+                "Use the official Docker image (ghcr.io/huggingface/text-generation-inference:latest) "
+                "or install flash attention with `cd server && make install install-flash-attention`"
+            ) from e
+        else:
+            for idx in range(torch.cuda.device_count()):
+                name = torch.cuda.get_device_name(idx)
+                if "MI210" not in name and "MI250" not in name:
+                    raise ImportError(
+                        f"AMD GPU {torch.cuda.get_device_name(idx)} does not support flash-attention"
+                    )
+            raise ImportError(
+                f"AMD GPU with ROCm capability {major} {minor} is not supported"
+            ) from e
+
+
+SUPPORTS_WINDOWING = False
+if ENGINE == "ck":
+
+    def attention(
+        q,
+        key_cache: torch.Tensor,
+        value_cache: torch.Tensor,
+        seqlen: Seqlen,
+        block_tables: torch.Tensor,
+        softmax_scale: float,
+        window_size_left: int = -1,
+        causal: bool = True,
+        softcap: float = 0.0,
+    ):
+        if window_size_left <= 0 and window_size_left != -1:
+            raise ValueError("`window_size_left` must be > 0 or -1")
+
+        out = torch.empty_like(q)
+
+        # We do not need to check window_size_left (not supported) here, so it is already checked ahead of time at model load.
+        return flash_attn_2_cuda.varlen_fwd(
+            q,
+            key_cache,
+            value_cache,
+            out,
+            seqlen.cu_seqlen_q,
+            seqlen.cu_seqlen_q,
+            None,
+            None,
+            None,
+            None,
+            seqlen.max_q,
+            seqlen.max_k,
+            0.0,
+            softmax_scale,
+            False,
+            causal,
+            window_size_left,
+            0,
+            softcap,
+            False,
+            None,
+        )[0]
+
+elif ENGINE == "triton":
+    from .flash_attn_triton import triton_attention
+
+    def attention(
+        q,
+        key_cache: torch.Tensor,
+        value_cache: torch.Tensor,
+        seqlen: Seqlen,
+        block_tables: torch.Tensor,
+        softmax_scale: float,
+        window_size_left: int = -1,
+        causal: bool = True,
+        softcap: Optional[float] = None,
+    ):
+        if softcap is not None:
+            raise NotImplementedError("softcap is only available with CK flash attn")
+
+        out = torch.empty_like(q)
+
+        # We do not need to check window_size_left (not supported) here, so it is already checked ahead of time at model load.
+        output, _ = triton_attention(
+            q,
+            key_cache,
+            value_cache,
+            out,
+            seqlen.cu_seqlen_q,
+            seqlen.cu_seqlen_q,
+            seqlen.max_q,
+            seqlen.max_k,
+            causal,
+            softmax_scale,
+        )
+        return output
+
+else:
+    raise RuntimeError(f"Unknown attention engine {ENGINE}")
diff --git a/server/text_generation_server/layers/awq/quantize/qmodule.py b/server/text_generation_server/layers/awq/quantize/qmodule.py
index ca8caf50..391371a5 100644
--- a/server/text_generation_server/layers/awq/quantize/qmodule.py
+++ b/server/text_generation_server/layers/awq/quantize/qmodule.py
@@ -1,6 +1,6 @@
 # Copied logic from https://github.com/mit-han-lab/llm-awq/blob/f084f40bd996f3cf3a0633c1ad7d9d476c318aaa/awq/quantize/qmodule.py
 
-import math
+from typing import Optional
 import torch
 import torch.nn as nn
 import awq_inference_engine  # with CUDA kernels
@@ -17,7 +17,9 @@ import awq_inference_engine  # with CUDA kernels
 
 
 class WQLinear(nn.Module):
-    def __init__(self, w_bit, group_size, qweight, qzeros, scales, bias):
+    def __init__(
+        self, w_bit, group_size, qweight, qzeros, scales, bias: Optional[torch.Tensor]
+    ):
         super().__init__()
 
         if w_bit not in [4]:
@@ -35,10 +37,7 @@ class WQLinear(nn.Module):
         self.qweight = qweight
         self.qzeros = qzeros
         self.scales = scales
-        if bias:
-            self.bias = bias
-        else:
-            self.bias = None
+        self.bias = bias
 
     @torch.no_grad()
     def forward(self, x):
diff --git a/server/text_generation_server/layers/bnb.py b/server/text_generation_server/layers/bnb.py
index ca39919c..791d9b6d 100644
--- a/server/text_generation_server/layers/bnb.py
+++ b/server/text_generation_server/layers/bnb.py
@@ -1,15 +1,17 @@
-import torch
-from loguru import logger
-from functools import lru_cache
+from dataclasses import dataclass
+
 import bitsandbytes as bnb
+import torch
 from bitsandbytes.nn import Int8Params, Params4bit
+from text_generation_server.utils.weights import UnquantizedWeight
 
 
-@lru_cache(1)
-def warn_deprecate_bnb():
-    logger.warning(
-        "Bitsandbytes 8bit is deprecated, using `eetq` is a drop-in replacement, and has much better performnce"
-    )
+@dataclass
+class BNBWeight(UnquantizedWeight):
+    weight: torch.Tensor
+
+    def get_linear(self, bias: torch.Tensor):
+        return Linear8bitLt(self.weight, bias, has_fp16_weights=False, threshold=6.0)
 
 
 class Linear8bitLt(torch.nn.Module):
@@ -70,6 +72,22 @@ class Linear8bitLt(torch.nn.Module):
         return out
 
 
+@dataclass
+class BNBFP4Weight(UnquantizedWeight):
+    weight: torch.Tensor
+
+    def get_linear(self, bias: torch.Tensor):
+        return Linear4bit(self.weight, bias, quant_type="fp4")
+
+
+@dataclass
+class BNBNF4Weight(UnquantizedWeight):
+    weight: torch.Tensor
+
+    def get_linear(self, bias: torch.Tensor):
+        return Linear4bit(self.weight, bias, quant_type="nf4")
+
+
 class Linear4bit(torch.nn.Module):
     def __init__(self, weight, bias, quant_type):
         super().__init__()
diff --git a/server/text_generation_server/layers/eetq.py b/server/text_generation_server/layers/eetq.py
index fd22b5c6..b1e5235a 100644
--- a/server/text_generation_server/layers/eetq.py
+++ b/server/text_generation_server/layers/eetq.py
@@ -1,5 +1,23 @@
+from dataclasses import dataclass
+
 import torch
 from EETQ import quant_weights, w8_a16_gemm
+from text_generation_server.utils.weights import UnquantizedWeight
+
+
+@dataclass
+class EETQWeight(UnquantizedWeight):
+    weight: torch.Tensor
+
+    def get_linear(self, bias: torch.Tensor):
+        try:
+            from text_generation_server.layers.eetq import EETQLinear
+
+            return EETQLinear(self.weight, bias)
+        except ImportError:
+            raise ImportError(
+                "Please install EETQ from https://github.com/NetEase-FuXi/EETQ"
+            )
 
 
 class EETQLinear(torch.nn.Module):
diff --git a/server/text_generation_server/layers/exl2.py b/server/text_generation_server/layers/exl2.py
new file mode 100644
index 00000000..a6e07f45
--- /dev/null
+++ b/server/text_generation_server/layers/exl2.py
@@ -0,0 +1,78 @@
+from dataclasses import dataclass
+from typing import List, Union
+
+import torch
+from text_generation_server.utils.weights import Weight, Weights, WeightsLoader
+
+
+@dataclass
+class Exl2Weight(Weight):
+    """
+    Exllama2 exl2 quantized weights.
+    """
+
+    q_weight: torch.Tensor
+    q_scale: torch.Tensor
+    q_invperm: torch.Tensor
+    q_scale_max: torch.Tensor
+    q_groups: torch.Tensor
+
+    def __post_init__(self):
+        self.q_scale_max /= 256
+        self.q_invperm = self.q_invperm.short()
+
+    @property
+    def device(self) -> torch.device:
+        return self.q_weight.device
+
+    def get_linear(self, bias: torch.Tensor):
+        from text_generation_server.layers.gptq import ExllamaQuantLinear
+
+        return ExllamaQuantLinear(self, bias)
+
+
+class Exl2WeightsLoader(WeightsLoader):
+    """Loader for exl2-quantized weights."""
+
+    def get_weights(self, weights: "Weights", prefix: str):
+        """
+        Get weights at the given prefix and apply without tensor paralllism.
+        """
+        try:
+            q_weight = weights.get_tensor(f"{prefix}.q_weight")
+        except RuntimeError:
+            raise RuntimeError(
+                "Cannot load `exl2`-quantized weight, make sure the model is already quantized."
+            )
+
+        q_scale = weights.get_tensor(f"{prefix}.q_scale")
+        q_invperm = weights.get_tensor(f"{prefix}.q_invperm")
+        q_scale_max = weights.get_tensor(f"{prefix}.q_scale_max")
+        q_groups = weights.get_tensor(f"{prefix}.q_groups")
+
+        return Exl2Weight(
+            q_weight=q_weight,
+            q_scale=q_scale,
+            q_invperm=q_invperm,
+            q_scale_max=q_scale_max,
+            q_groups=q_groups,
+        )
+
+    def get_weights_col_packed(
+        self,
+        weights: Weights,
+        prefix: str,
+        block_sizes: Union[int, List[int]],
+    ):
+        raise RuntimeError("Column-packed weights are not supported for exl")
+
+    def get_weights_col(self, weights: Weights, prefix: str):
+        # Sharding is not yet supported, so we return the weights as-is.
+        return self.get_weights(weights, prefix)
+
+    def get_multi_weights_col(self, weights: Weights, prefixes: List[str], dim: int):
+        raise ValueError("get_multi_weights_col is not supported for exl2")
+
+    def get_weights_row(self, weights: Weights, prefix: str):
+        # Sharding is not yet supported, so we return the weights as-is.
+        return self.get_weights(weights, prefix)
diff --git a/server/text_generation_server/layers/fp8.py b/server/text_generation_server/layers/fp8.py
index dd61d081..61dd5115 100644
--- a/server/text_generation_server/layers/fp8.py
+++ b/server/text_generation_server/layers/fp8.py
@@ -1,12 +1,69 @@
 import torch
 
+from dataclasses import dataclass
+from typing import Optional, Union, List
+from loguru import logger
+
+from text_generation_server.utils.import_utils import SYSTEM
+from text_generation_server.utils.weights import (
+    Weight,
+    WeightsLoader,
+    UnquantizedWeight,
+    Weights,
+)
+from text_generation_server.utils.log import log_master, log_once
+import importlib.util
+
+
+FBGEMM_MM_AVAILABLE = False
+FBGEMM_DYN_AVAILABLE = False
+
+
+def is_fbgemm_gpu_available():
+    try:
+        return importlib.util.find_spec("fbgemm_gpu.experimental.gen_ai") is not None
+    except ModuleNotFoundError:
+        return False
+
+
+if is_fbgemm_gpu_available():
+    if SYSTEM == "cuda":
+        major, _ = torch.cuda.get_device_capability()
+        FBGEMM_MM_AVAILABLE = major == 9
+        FBGEMM_DYN_AVAILABLE = major >= 8
+else:
+    log_master(logger.warning, "FBGEMM fp8 kernels are not installed.")
+
+
+def get_fp8_linear() -> torch.nn.Module:
+    """
+    Return an FP8 linear `Module` that is compatible with the current system.
+    """
+
+    if SYSTEM == "cuda":
+        major, _ = torch.cuda.get_device_capability()
+        if major == 8:
+            from text_generation_server.layers.marlin import GPTQMarlinFP8Linear
+
+            return GPTQMarlinFP8Linear
+
+    # On other systems let Torch decide if the hardware supports FP8.
+    return Fp8Linear
+
+
+def fp8_quantize(
+    weight, scale_upper_bound=None, qdtype=torch.float8_e4m3fn, scalar=False
+):
+    if FBGEMM_DYN_AVAILABLE and not scalar:
+        qweight, scale = torch.ops.fbgemm.quantize_fp8_per_row(
+            weight, bs=None, scale_ub=scale_upper_bound, output_dtype=qdtype
+        )
+        return qweight, scale
 
-def fp8_quantize(weight, qdtype=torch.float8_e4m3fn):
-    device = weight.device
     # weight, scale = quant_weights(weight, torch.int8, False)
     finfo = torch.finfo(qdtype)
     # Calculate the scale as dtype max divided by absmax
-    scale = finfo.max / weight.abs().max().clamp(min=1e-12)
+    scale = finfo.max / weight.abs().max().clamp(min=1e-12, max=scale_upper_bound)
     # scale and clamp the tensor to bring it to
     # the representative range of float8 data type
     # (as default cast is unsaturated)
@@ -18,20 +75,198 @@ def fp8_quantize(weight, qdtype=torch.float8_e4m3fn):
     return qweight, scale
 
 
+class HybridFP8UnquantLoader(WeightsLoader):
+    """Weight loader that loads FP8 and unquantized Torch tensors."""
+
+    def __init__(self, activation_scale_ub: Optional[float], to_fp8: bool):
+        self.activation_scale_ub = activation_scale_ub
+        self.to_fp8 = to_fp8
+
+    def get_weights(self, weights: "Weights", prefix: str):
+        w = weights.get_tensor(f"{prefix}.weight")
+
+        if w.dtype == torch.float8_e4m3fn:
+            # FP8 branch
+            scale = (
+                weights.get_tensor(f"{prefix}.weight_scale", to_dtype=False)
+                .reshape(-1)
+                .expand(w.shape[0])
+            )
+            return Fp8Weight(
+                weight=w,
+                weight_scale=scale,
+                activation_scale_ub=self.activation_scale_ub,
+                dtype=weights.dtype,
+            )
+        if self.to_fp8:
+            return Fp8Weight(weight=w, dtype=weights.dtype)
+
+        return UnquantizedWeight(w)
+
+    def get_weights_col_packed(
+        self,
+        weights: Weights,
+        prefix: str,
+        block_sizes: Union[int, List[int]],
+    ):
+        w = weights.get_packed_sharded(
+            f"{prefix}.weight", dim=0, block_sizes=block_sizes
+        )
+
+        if w.dtype == torch.float8_e4m3fn:
+            # FP8 branch
+            scale = weights.get_tensor(f"{prefix}.weight_scale", to_dtype=False)
+            if scale.numel() > 1:
+                scale = weights.get_packed_sharded(
+                    f"{prefix}.weight_scale",
+                    dim=0,
+                    block_sizes=block_sizes,
+                    to_dtype=False,
+                )
+            scale = scale.reshape(-1).expand(w.shape[0])
+
+            return Fp8Weight(
+                weight=w,
+                weight_scale=scale,
+                activation_scale_ub=self.activation_scale_ub,
+                dtype=weights.dtype,
+            )
+        if self.to_fp8:
+            return Fp8Weight(weight=w, dtype=weights.dtype)
+
+        return UnquantizedWeight(w)
+
+    def get_multi_weights_col(self, weights: "Weights", prefixes: List[str], dim: int):
+        # FIXME: Force to_device to false as fp8 weights do not support torch.cat on device yet
+        w = [
+            weights.get_sharded(f"{p}.weight", dim=0, to_device=False) for p in prefixes
+        ]
+        shapes = [x.shape for x in w]
+
+        # Concat then send to the device
+        w = torch.cat(w, dim=dim).to(weights.device)
+
+        # FP8 branch
+        if w.dtype == torch.float8_e4m3fn:
+            scale = [
+                _load_scalar_or_matrix_scale(weights, f"{p}.weight_scale", shape)
+                for p, shape in zip(prefixes, shapes)
+            ]
+            scale = torch.cat(scale, dim=0).reshape(-1)
+
+            return Fp8Weight(
+                weight=w,
+                weight_scale=scale,
+                activation_scale_ub=self.activation_scale_ub,
+                dtype=weights.dtype,
+            )
+        if self.to_fp8:
+            return Fp8Weight(weight=w, dtype=weights.dtype)
+
+        return UnquantizedWeight(w)
+
+    def get_weights_row(self, weights: "Weights", prefix: str):
+        w = weights.get_sharded(f"{prefix}.weight", dim=1)
+        # FP8 branch
+        if w.dtype == torch.float8_e4m3fn:
+            scale = (
+                weights.get_tensor(f"{prefix}.weight_scale", to_dtype=False)
+                .reshape(-1)
+                .expand(w.shape[0])
+            )
+            return Fp8Weight(
+                weight=w,
+                weight_scale=scale,
+                activation_scale_ub=self.activation_scale_ub,
+                dtype=weights.dtype,
+            )
+        if self.to_fp8:
+            return Fp8Weight(weight=w, dtype=weights.dtype)
+
+        return UnquantizedWeight(w)
+
+
+@dataclass
+class Fp8Weight(Weight):
+    weight: torch.Tensor
+    dtype: torch.dtype
+    weight_scale: Optional[torch.Tensor] = None
+    activation_scale_ub: Optional[float] = None
+
+    def get_linear(self, bias: torch.Tensor):
+        if self.weight_scale is None:
+            return get_fp8_linear().from_unquant(self.weight, bias, self.dtype)
+        # This is not checked by the fbgemm kernels, but they require contiguous
+        # memory. Can be non-contiguous when we e.g. expand from scalars.
+        self.weight_scale = self.weight_scale.contiguous()
+        return get_fp8_linear().from_fp8(
+            self.weight, self.weight_scale, self.activation_scale_ub, bias, self.dtype
+        )
+
+
 class Fp8Linear(torch.nn.Module):
     def __init__(
         self,
-        weight,
+        qweight,
+        scale,
+        scale_upper_bound,
         bias,
+        dtype,
     ) -> None:
         super().__init__()
-        self.dtype = weight.dtype
-        self.qweight, self.scale = fp8_quantize(weight)
+        if FBGEMM_MM_AVAILABLE:
+            log_once(logger.info, "Using FBGEMM fp8 optimized kernels")
+
+        self.dtype = dtype
+        self.qweight = qweight
+        self.scale = scale
+        self.scale_upper_bound = (
+            torch.tensor(
+                [scale_upper_bound], dtype=torch.float32, device=qweight.device
+            )
+            if scale_upper_bound is not None
+            else None
+        )
 
         self.bias = bias if bias is not None else None
 
+    @classmethod
+    def from_unquant(cls, weight, bias, dtype):
+        qweight, scale = fp8_quantize(weight, scalar=not FBGEMM_MM_AVAILABLE)
+        return cls(
+            qweight=qweight, scale=scale, scale_upper_bound=None, bias=bias, dtype=dtype
+        )
+
+    @classmethod
+    def from_fp8(cls, weight, scale, input_scale, bias, dtype):
+        if FBGEMM_DYN_AVAILABLE:
+            # fbgemm needs float32 scales.
+            scale = scale.float()
+        return cls(
+            qweight=weight,
+            scale=scale,
+            scale_upper_bound=input_scale,
+            bias=bias,
+            dtype=dtype,
+        )
+
     def forward(self, input: torch.Tensor) -> torch.Tensor:
-        qinput, scale = fp8_quantize(input)
+        if FBGEMM_MM_AVAILABLE:
+            qinput, scale = fp8_quantize(
+                input, scale_upper_bound=self.scale_upper_bound
+            )
+
+            y = torch.ops.fbgemm.f8f8bf16_rowwise(
+                qinput,
+                self.qweight,
+                scale,
+                self.scale,
+                use_fast_accum=True,
+                bias=self.bias,
+            )
+            return y.to(self.dtype)
+
+        qinput, scale = fp8_quantize(input, scalar=True)
         output, _ = torch._scaled_mm(
             qinput,
             self.qweight.t(),
@@ -41,3 +276,10 @@ class Fp8Linear(torch.nn.Module):
             bias=self.bias,
         )
         return output
+
+
+def _load_scalar_or_matrix_scale(weights: Weights, prefix: str, shape: torch.Size):
+    scale = weights.get_tensor(prefix, to_dtype=False)
+    if scale.numel() > 1:
+        scale = weights.get_sharded(prefix, dim=0, to_dtype=False)
+    return scale.reshape(-1).expand(shape[0])
diff --git a/server/text_generation_server/layers/gptq/__init__.py b/server/text_generation_server/layers/gptq/__init__.py
index 1c46f493..505caa59 100644
--- a/server/text_generation_server/layers/gptq/__init__.py
+++ b/server/text_generation_server/layers/gptq/__init__.py
@@ -1,9 +1,412 @@
 import os
-import torch
-from text_generation_server.utils.import_utils import (
-    SYSTEM,
-)
+from dataclasses import dataclass
+from typing import List, Optional, Union
 
+import torch
+from loguru import logger
+from text_generation_server.utils.import_utils import SYSTEM
+from text_generation_server.utils.log import log_once
+from text_generation_server.utils.weights import Weight, Weights, WeightsLoader
+
+
+@dataclass
+class GPTQWeight(Weight):
+    qweight: torch.Tensor
+    qzeros: torch.Tensor
+    scales: torch.Tensor
+    g_idx: Optional[torch.Tensor]
+    bits: int
+    groupsize: int
+    use_awq_kernel: bool
+    use_exllama: bool
+
+    def __post_init__(self):
+        if self.scales.dtype == torch.float:
+            self.scales = self.scales.half()
+
+    @property
+    def device(self) -> torch.device:
+        return self.qweight.device
+
+    def get_linear(self, bias: torch.Tensor):
+        if self.use_awq_kernel:
+            if SYSTEM == "rocm":
+                raise NotImplementedError(
+                    "AWQ GEMM kernel can't be used on ROCm systems, please use `--quantize gptq` instead "
+                    "to use Exllama/GPTQ kernels for AWQ inference."
+                )
+            try:
+                from text_generation_server.layers.awq.quantize.qmodule import WQLinear
+
+                return WQLinear(
+                    w_bit=self.bits,
+                    group_size=self.groupsize,
+                    qweight=self.qweight,
+                    qzeros=self.qzeros,
+                    scales=self.scales,
+                    bias=bias,
+                )
+            except ImportError:
+                raise NotImplementedError(
+                    "You do not seem to have awq installed, either install it (cd server &&  make install-awq), or try using GPTQ `---quantize gptq` a conversion AWQ->GPTQ will happen on the fly"
+                )
+        elif self.use_exllama:
+            try:
+                from text_generation_server.layers.gptq import ExllamaQuantLinear
+            except ImportError:
+                raise NotImplementedError(
+                    "Exllama gptq kernels are not installed. Install them `cd server/exllama_kernels && python setup.py install && cd ../exllamav2_kernels && python setup.py install`"
+                )
+
+            return ExllamaQuantLinear(self, bias)
+        else:
+            from text_generation_server.layers.gptq.quant_linear import QuantLinear
+
+            return QuantLinear(
+                self.qweight,
+                self.qzeros,
+                self.scales,
+                self.g_idx,
+                bias,
+                self.bits,
+                self.groupsize,
+            )
+
+
+class GPTQWeightsLoader(WeightsLoader):
+    """
+    Loader for GPTQ- and AWQ-quantized weights.
+    """
+
+    def __init__(
+        self,
+        *,
+        bits: int,
+        desc_act: bool,
+        groupsize: int,
+        quant_method: str,
+        quantize: str,
+        sym: bool,
+    ):
+        self.bits = bits
+        self.desc_act = desc_act
+        self.groupsize = groupsize
+        self.quant_method = quant_method
+        self.quantize = quantize
+        self.sym = sym
+
+    def get_weights(self, weights: Weights, prefix: str):
+        self._get_gptq_params(weights)
+
+        use_exllama = True
+        if self.bits != 4:
+            use_exllama = False
+
+        if self.desc_act:
+            log_once(logger.warning, "Disabling exllama because desc_act=True")
+            use_exllama = False
+
+        try:
+            qweight = weights.get_tensor(f"{prefix}.qweight")
+        except RuntimeError:
+            raise RuntimeError(
+                "Cannot load `gptq` weight, make sure the model is already quantized, or quantize it with `text-generation-server quantize ORIGINAL_MODEL_ID NEW_MODEL_ID`"
+            )
+
+        if self.quantize == "gptq" and self.quant_method == "gptq":
+            g_idx = weights.get_tensor(f"{prefix}.g_idx")
+        else:
+            g_idx = None
+
+        from text_generation_server.layers.gptq import (
+            HAS_EXLLAMA,
+            CAN_EXLLAMA,
+            GPTQWeight,
+        )
+
+        if use_exllama:
+            if not HAS_EXLLAMA:
+                if CAN_EXLLAMA:
+                    log_once(
+                        logger.warning,
+                        "Exllama GPTQ cuda kernels (which are faster) could have been used, but are not currently installed, try using BUILD_EXTENSIONS=True",
+                    )
+                use_exllama = False
+            else:
+                log_once(logger.info, f"Using exllama kernels v{HAS_EXLLAMA}")
+
+        qzeros = weights.get_tensor(f"{prefix}.qzeros")
+        scales = weights.get_tensor(f"{prefix}.scales")
+
+        if use_exllama and g_idx is not None:
+            g_idx = g_idx - g_idx[0]
+
+        if self.quantize == "gptq" and self.quant_method == "awq":
+            log_once(
+                logger.info, "Converting AWQ model to Exllama/GPTQ packing format."
+            )
+            from text_generation_server.layers.awq.conversion_utils import (
+                fast_awq_to_gptq,
+            )
+
+            qweight, qzeros = fast_awq_to_gptq(qweight, qzeros)
+            if use_exllama:
+                g_idx = None
+            else:
+                g_idx = (
+                    torch.arange(
+                        qweight.shape[0] * (32 // self.bits),
+                        device=qweight.device,
+                    )
+                    // self.groupsize
+                ).to(dtype=torch.int32)
+
+        return GPTQWeight(
+            qweight=qweight,
+            qzeros=qzeros,
+            scales=scales,
+            g_idx=g_idx,
+            bits=self.bits,
+            groupsize=self.groupsize,
+            use_exllama=use_exllama,
+        )
+
+    def get_weights_col_packed(
+        self,
+        weights: Weights,
+        prefix: str,
+        block_sizes: Union[int, List[int]],
+    ):
+        try:
+            qweight = weights.get_packed_sharded(
+                f"{prefix}.qweight", dim=1, block_sizes=block_sizes
+            )
+        except RuntimeError:
+            raise RuntimeError(
+                f"Cannot load `{self.quantize}` weight, make sure the model is already quantized."
+            )
+        scales = weights.get_packed_sharded(
+            f"{prefix}.scales", dim=1, block_sizes=block_sizes
+        )
+        scales = scales.to(dtype=weights.dtype)
+
+        self._get_gptq_params(weights)
+
+        qzeros = weights.get_packed_sharded(
+            f"{prefix}.qzeros", dim=1, block_sizes=block_sizes
+        )
+        if self.quantize == "gptq" and self.quant_method == "gptq":
+            g_idx = weights.get_tensor(f"{prefix}.g_idx")
+        elif self.quantize == "gptq" and self.quant_method == "awq":
+            log_once(
+                logger.info, "Converting AWQ model to Exllama/GPTQ packing format."
+            )
+            from text_generation_server.layers.awq.conversion_utils import (
+                fast_awq_to_gptq,
+            )
+
+            qweight, qzeros = fast_awq_to_gptq(qweight, qzeros)
+            g_idx = (
+                torch.arange(
+                    qweight.shape[0] * (32 // self.bits),
+                    device=qweight.device,
+                )
+                // self.groupsize
+            ).to(dtype=torch.int32)
+        else:
+            g_idx = None
+
+        return GPTQWeight(
+            qweight=qweight,
+            qzeros=qzeros,
+            scales=scales,
+            g_idx=g_idx,
+            bits=self.bits,
+            groupsize=self.groupsize,
+            use_awq_kernel=self.quantize == "awq",
+            use_exllama=False,
+        )
+
+    def get_multi_weights_col(self, weights: Weights, prefixes: List[str], dim: int):
+        try:
+            qweight = torch.cat(
+                [weights.get_sharded(f"{p}.qweight", dim=1) for p in prefixes], dim=1
+            )
+        except RuntimeError:
+            raise RuntimeError(
+                f"Cannot load `{self.quantize}` weight, make sure the model is already quantized"
+            )
+
+        scales = torch.cat(
+            [weights.get_sharded(f"{p}.scales", dim=1) for p in prefixes], dim=1
+        )
+
+        self._get_gptq_params(weights)
+
+        qzeros = torch.cat(
+            [weights.get_sharded(f"{p}.qzeros", dim=1) for p in prefixes], dim=1
+        )
+
+        from text_generation_server.layers.gptq import HAS_EXLLAMA
+
+        use_exllama = (
+            self.bits == 4
+            and HAS_EXLLAMA
+            and self.quantize == "gptq"
+            and not self.desc_act
+        )
+
+        if self.quantize == "gptq" and self.quant_method == "gptq":
+            w = [weights.get_tensor(f"{p}.g_idx") for p in prefixes]
+            for w2 in w[1:]:
+                torch.testing.assert_close(w2, w[0])
+            g_idx = w[0]
+        elif self.quantize == "gptq" and self.quant_method == "awq":
+            log_once(
+                logger.info, "Converting AWQ model to Exllama/GPTQ packing format."
+            )
+            from text_generation_server.layers.awq.conversion_utils import (
+                fast_awq_to_gptq,
+            )
+
+            qweight, qzeros = fast_awq_to_gptq(qweight, qzeros)
+            if use_exllama:
+                g_idx = None
+            else:
+                g_idx = (
+                    torch.arange(
+                        qweight.shape[0] * (32 // self.bits),
+                        device=qweight.device,
+                    )
+                    // self.groupsize
+                ).to(dtype=torch.int32)
+        else:
+            g_idx = None
+
+        return GPTQWeight(
+            qweight=qweight,
+            qzeros=qzeros,
+            scales=scales,
+            g_idx=g_idx,
+            bits=self.bits,
+            groupsize=self.groupsize,
+            use_awq_kernel=self.quantize == "awq",
+            use_exllama=use_exllama,
+        )
+
+    def get_weights_row(self, weights: Weights, prefix: str):
+        self._get_gptq_params(weights)
+
+        use_exllama = True
+        if self.bits != 4:
+            use_exllama = False
+
+        if self.desc_act:
+            log_once(logger.warning, "Disabling exllama because desc_act=True")
+            use_exllama = False
+
+        try:
+            qweight = weights.get_sharded(f"{prefix}.qweight", dim=0)
+        except RuntimeError:
+            raise RuntimeError(
+                "Cannot load `gptq` weight, make sure the model is already quantized, or quantize it with `text-generation-server quantize ORIGINAL_MODEL_ID NEW_MODEL_ID`"
+            )
+
+        if self.quantize == "gptq" and self.quant_method == "gptq":
+            g_idx = weights.get_sharded(f"{prefix}.g_idx", dim=0)
+        else:
+            g_idx = None
+
+        if weights.process_group.size() > 1:
+            if g_idx is not None:
+                if (
+                    not torch.equal(
+                        g_idx.cpu(),
+                        torch.tensor(
+                            [i // self.groupsize for i in range(g_idx.shape[0])],
+                            dtype=torch.int32,
+                        ),
+                    )
+                    and not (g_idx == 0).all()
+                ):
+                    # Exllama implementation does not support row tensor parallelism with act-order, as
+                    # it would require to reorder input activations that are split unto several GPUs
+                    use_exllama = False
+
+        from text_generation_server.layers.gptq import (
+            CAN_EXLLAMA,
+            HAS_EXLLAMA,
+            GPTQWeight,
+        )
+
+        if use_exllama:
+            if not HAS_EXLLAMA:
+                if CAN_EXLLAMA:
+                    log_once(
+                        logger.warning,
+                        "Exllama GPTQ cuda kernels (which are faster) could have been used, but are not currently installed, try using BUILD_EXTENSIONS=True",
+                    )
+                use_exllama = False
+            else:
+                log_once(logger.info, f"Using exllama kernels v{HAS_EXLLAMA}")
+
+        if use_exllama and self.groupsize != -1:
+            qzeros = weights.get_sharded(f"{prefix}.qzeros", dim=0)
+            scales = weights.get_sharded(f"{prefix}.scales", dim=0)
+        else:
+            qzeros = weights.get_tensor(f"{prefix}.qzeros")
+            scales = weights.get_tensor(f"{prefix}.scales")
+
+        if use_exllama and g_idx is not None:
+            g_idx = g_idx - g_idx[0]
+
+        if self.quantize == "gptq" and self.quant_method == "awq":
+            log_once(
+                logger.info, "Converting AWQ model to Exllama/GPTQ packing format."
+            )
+            from text_generation_server.layers.awq.conversion_utils import (
+                fast_awq_to_gptq,
+            )
+
+            qweight, qzeros = fast_awq_to_gptq(qweight, qzeros)
+            if use_exllama:
+                g_idx = None
+            else:
+                g_idx = (
+                    torch.arange(
+                        qweight.shape[0] * (32 // self.bits),
+                        device=qweight.device,
+                    )
+                    // self.groupsize
+                ).to(dtype=torch.int32)
+
+        return GPTQWeight(
+            qweight=qweight,
+            qzeros=qzeros,
+            scales=scales,
+            g_idx=g_idx,
+            bits=self.bits,
+            groupsize=self.groupsize,
+            use_awq_kernel=self.quantize == "awq",
+            use_exllama=use_exllama,
+        )
+
+    def _get_gptq_params(self, weights: Weights):
+        if weights._has_tensor("gptq_bits") and weights._has_tensor("gptq_groupsize"):
+            self.bits = weights.get_tensor("gptq_bits").item()
+            self.groupsize = weights.get_tensor("gptq_groupsize").item()
+            self.desc_act = False
+            # `server quantize` used asymmetric quantization unconditionally
+            # before the `gptq_sym` setting tensor was added.
+            self.sym = (
+                weights.get_tensor("gptq_sym").item()
+                if weights._has_tensor("gptq_sym")
+                else False
+            )
+            self.quant_method = "gptq"
+
+
+# Needs to be at the end because circular import.
 try:
     major, _minor = torch.cuda.get_device_capability()
 except Exception:
@@ -18,22 +421,20 @@ elif CAN_EXLLAMA:
     try:
         if V2:
             from text_generation_server.layers.gptq.exllamav2 import (
-                QuantLinear as ExllamaQuantLinear,
-                create_exllama_buffers,
-                set_device,
+                QuantLinear as ExllamaQuantLinear,  # noqa: F401
+                create_exllama_buffers,  # noqa: F401
+                set_device,  # noqa: F401
             )
 
             HAS_EXLLAMA = "2"
         else:
             from text_generation_server.layers.gptq.exllama import (
-                Ex4bitLinear as ExllamaQuantLinear,
-                create_exllama_buffers,
-                set_device,
+                Ex4bitLinear as ExllamaQuantLinear,  # noqa: F401
+                create_exllama_buffers,  # noqa: F401
+                set_device,  # noqa: F401
             )
 
             HAS_EXLLAMA = "1"
 
     except ImportError:
         pass
-
-from text_generation_server.layers.gptq.quant_linear import QuantLinear
diff --git a/server/text_generation_server/layers/gptq/custom_autotune.py b/server/text_generation_server/layers/gptq/custom_autotune.py
index 1eb40f1e..0388ef20 100644
--- a/server/text_generation_server/layers/gptq/custom_autotune.py
+++ b/server/text_generation_server/layers/gptq/custom_autotune.py
@@ -91,7 +91,7 @@ class Autotuner(triton.KernelInterface):
                 kernel_call, quantiles=(0.5, 0.2, 0.8), rep=40
             )
         except triton.OutOfResources:
-            return (float("inf"), float("inf"), float("inf"))
+            return [float("inf"), float("inf"), float("inf")]
 
     def run(self, *args, **kwargs):
         self.nargs = dict(zip(self.arg_names, args))
diff --git a/server/text_generation_server/layers/gptq/exllama.py b/server/text_generation_server/layers/gptq/exllama.py
index 32f817db..f27666b7 100644
--- a/server/text_generation_server/layers/gptq/exllama.py
+++ b/server/text_generation_server/layers/gptq/exllama.py
@@ -1,3 +1,4 @@
+from text_generation_server.layers.gptq import GPTQWeight
 import torch
 from exllama_kernels import make_q4, q4_matmul, prepare_buffers, set_tuning_params
 
@@ -65,24 +66,25 @@ def create_exllama_buffers(max_total_tokens: int):
 class Ex4bitLinear(torch.nn.Module):
     """Linear layer implementation with per-group 4-bit quantization of the weights"""
 
-    def __init__(self, qweight, qzeros, scales, g_idx, bias, bits, groupsize):
+    def __init__(self, weight: GPTQWeight, bias):
         super().__init__()
         global MAX_DQ, MAX_INNER, ACT_ORDER, DEVICE
-        assert bits == 4
+        assert weight.bits == 4
 
-        self.device = qweight.device
-        self.qweight = qweight
-        self.qzeros = qzeros
-        self.scales = scales
-        self.g_idx = g_idx.cpu() if g_idx is not None else None
+        self.device = weight.qweight.device
+        self.qweight = weight.qweight
+        self.qzeros = weight.qzeros
+        self.scales = weight.scales
+        self.g_idx = weight.g_idx.cpu() if weight.g_idx is not None else None
         self.bias = bias if bias is not None else None
 
         if self.g_idx is not None and (
             (self.g_idx == 0).all()
             or torch.equal(
-                g_idx.cpu(),
+                weight.g_idx.cpu(),
                 torch.tensor(
-                    [i // groupsize for i in range(g_idx.shape[0])], dtype=torch.int32
+                    [i // weight.groupsize for i in range(weight.g_idx.shape[0])],
+                    dtype=torch.int32,
                 ),
             )
         ):
@@ -96,8 +98,8 @@ class Ex4bitLinear(torch.nn.Module):
             self.qweight, self.qzeros, self.scales, self.g_idx, self.device.index
         )
 
-        self.height = qweight.shape[0] * 8
-        self.width = qweight.shape[1]
+        self.height = weight.qweight.shape[0] * 8
+        self.width = weight.qweight.shape[1]
 
         # Infer groupsize from height of qzeros
         self.groupsize = None
@@ -105,7 +107,7 @@ class Ex4bitLinear(torch.nn.Module):
             self.groupsize = (self.qweight.shape[0] * 8) // (self.qzeros.shape[0])
 
         if self.groupsize is not None:
-            assert groupsize == self.groupsize
+            assert weight.groupsize == self.groupsize
 
         # Handle act-order matrix
         if self.g_idx is not None:
diff --git a/server/text_generation_server/layers/gptq/exllamav2.py b/server/text_generation_server/layers/gptq/exllamav2.py
index 80836a95..920a6adf 100644
--- a/server/text_generation_server/layers/gptq/exllamav2.py
+++ b/server/text_generation_server/layers/gptq/exllamav2.py
@@ -1,20 +1,38 @@
 # Adapted from turboderp exllama: https://github.com/turboderp/exllamav2
 
+from dataclasses import dataclass
+from typing import Optional
 import torch
 import torch.nn as nn
 
 from loguru import logger
 
+from text_generation_server.layers.exl2 import Exl2Weight
+from text_generation_server.layers.gptq import GPTQWeight
+from text_generation_server.utils.log import log_master
+
 try:
-    from exllamav2_kernels import make_q_matrix, gemm_half_q_half
+    from exllamav2.ext import exllamav2_ext
+
+    make_q_matrix = exllamav2_ext.make_q_matrix
+    gemm_half_q_half = exllamav2_ext.gemm_half_q_half
 except ImportError:
-    logger.error("exllamav2_kernels not installed.")
+    log_master(logger.warning, "exllamav2_kernels not installed.")
     raise
 
 # Dummy tensor to pass instead of g_idx since there is no way to pass "None" to a C++ extension
 none_tensor = torch.empty((1, 1), device="meta")
 
 
+@dataclass
+class _ExtraTensors:
+    """Additional generated quantizer tensors."""
+
+    q_group_map: Optional[torch.Tensor] = None
+    q_invperm: Optional[torch.Tensor] = None
+    q_perm: Optional[torch.Tensor] = None
+
+
 def ext_gemm_half_q_half(x, q_handle, q4_width, force_cuda):
     """Matrix multiplication, returns x @ q4"""
     output_shape = x.shape[:-1] + (q4_width,)
@@ -24,11 +42,7 @@ def ext_gemm_half_q_half(x, q_handle, q4_width, force_cuda):
     return output.view(output_shape)
 
 
-# Group map needed for irregular group sizes
-
-
-def make_group_map(q_groups, num_qrows):
-
+def make_group_map(q_groups: torch.Tensor, num_qrows: int):
     gr = q_groups.tolist()
     group_map = []
     num_groups = len(gr) // 2
@@ -50,79 +64,90 @@ def make_group_map(q_groups, num_qrows):
 # Create Q matrix
 
 
-def ext_make_q_matrix(w: dict, temp_dq, key: str = None):
+def ext_make_q_matrix(
+    w: Exl2Weight | GPTQWeight,
+    extra: _ExtraTensors,
+    temp_dq,
+    key: Optional[str] = None,
+):
     """
     Create Q matrix
     """
-    # EXL2
-    # won't work as the moment because the tensors are not the same.
-    if "q_weight" in w:
-        w["q_scale_max"] /= 256
-        w["q_perm"] = w["q_perm"].short()
-        w["q_invperm"] = w["q_invperm"].short()
+    # max_dq_size = 512*(1024**2)
+    # max_dq_rows = max_dq_size // out_features[0]
+    max_dq_rows = 0
 
-        if "q_group_map" not in w:
-            w["q_group_map"] = make_group_map(w["q_groups"], w["q_weight"].shape[0])
+    # EXL2
+    if isinstance(w, Exl2Weight):
+        extra.q_group_map = make_group_map(w.q_groups, w.q_weight.shape[0])
+        extra.q_perm = torch.argsort(w.q_invperm).short()
 
         return make_q_matrix(
-            w["q_weight"],
-            w["q_perm"],
-            w["q_invperm"],
-            w["q_scale"],
-            w["q_scale_max"],
-            w["q_groups"],
-            w["q_group_map"],
-            none_tensor,
-            none_tensor,
-            none_tensor,
+            w.q_weight,
+            extra.q_perm,
+            w.q_invperm,
+            w.q_scale,
+            w.q_scale_max,
+            w.q_groups,
+            extra.q_group_map,
+            none_tensor,  # zeros
+            none_tensor,  # scales
+            none_tensor,  # g_idx
+            none_tensor,  # bias
             temp_dq,
+            max_dq_rows,
         )
     # GPTQ
-    elif "qweight" in w:
-        if w["scales"].dtype == torch.float:
-            w["scales"] = w["scales"].half()
+    elif isinstance(w, GPTQWeight):
+        if w.scales.dtype == torch.float:
+            w.scales = w.scales.half()
 
         # GPTQ with g_idx (act_order)
-        if w.get("g_idx", None) is not None and not (w["g_idx"] == 0).all().item():
-            w["q_perm"] = torch.empty(
-                (w["qweight"].shape[0] * 8,),
+        if w.g_idx is not None and not (w.g_idx == 0).all().item():
+            extra.q_perm = torch.empty(
+                (w.qweight.shape[0] * 8,),
                 dtype=torch.short,
-                device=w["qweight"].device,
+                device=w.qweight.device,
             )
-            w["q_invperm"] = torch.empty_like(w["q_perm"])
+            extra.q_invperm = torch.empty_like(extra.q_perm)
             # make_q4 segfaults if g_idx is not on cpu in the act-order case. In the non act-order case, None needs to be passed for g_idx.
             return make_q_matrix(
-                w["qweight"],
-                w["q_perm"],
-                w["q_invperm"],
-                none_tensor,
-                none_tensor,
-                none_tensor,
-                none_tensor,
-                w["qzeros"],
-                w["scales"],
-                w["g_idx"].cpu(),
+                w.qweight,
+                extra.q_perm,
+                extra.q_invperm,
+                none_tensor,  # q_scale
+                none_tensor,  # q_scale_max
+                none_tensor,  # q_groups
+                none_tensor,  # q_group_map
+                w.qzeros,
+                w.scales,
+                w.g_idx.cpu(),
+                none_tensor,  # bias
                 temp_dq,
+                max_dq_rows,
             )
         # GPTQ without g_idx
         else:
             return make_q_matrix(
-                w["qweight"],
-                none_tensor,
-                none_tensor,
-                none_tensor,
-                none_tensor,
-                none_tensor,
-                none_tensor,
-                w["qzeros"],
-                w["scales"],
-                none_tensor,
+                w.qweight,
+                none_tensor,  # q_perm
+                none_tensor,  # q_invperm
+                none_tensor,  # q_scale
+                none_tensor,  # q_scale_max
+                none_tensor,  # q_groups
+                none_tensor,  # q_group_map
+                w.qzeros,
+                w.scales,
+                none_tensor,  # g_idx
+                none_tensor,  # bias
                 temp_dq,
+                max_dq_rows,
             )
+    else:
+        RuntimeError("Cannot create handle")
 
 
 DEVICE = None
-FIXED_BYTES = 0
 LAYERS = []
 
 
@@ -132,8 +157,19 @@ def set_device(device):
 
 
 def create_exllama_buffers(max_total_tokens: int):
-    global FIXED_BYTES, LAYERS, DEVICE
-    temp_dq = ExLlamaV2DeviceTensors(DEVICE, FIXED_BYTES)
+    global LAYERS, DEVICE
+
+    # No need to initialize scratch space if there are no layers
+    # that use ExLLamav2.
+    if len(LAYERS) == 0:
+        return
+
+    # Find the size of the scratch space.
+    scratch_bytes = max(
+        layer.scratch_space_fixed(max_input_len=max_total_tokens, max_batch_size=1)
+        for layer in LAYERS
+    )
+    temp_dq = ExLlamaV2DeviceTensors(DEVICE, scratch_bytes)
 
     for layer in LAYERS:
         layer.post_init(temp_dq)
@@ -144,49 +180,48 @@ class QuantLinear(nn.Module):
 
     """Linear layer implementation with per-group 4-bit quantization of the weights"""
 
-    # def __init__(self, bits, group_size, infeatures, outfeatures, bias, trainable=False, **kwargs):
-    def __init__(self, qweight, qzeros, scales, g_idx, bias, bits, groupsize):
+    def __init__(
+        self,
+        weight: Exl2Weight | GPTQWeight,
+        bias: torch.Tensor,
+    ):
         super().__init__()
-        if bits != 4:
-            raise ValueError(
-                f"Exllamav2 kernel supports only bits=4, requested bits={bits}. Something is wrong in the model initialization."
-            )
+
         self.q_handle = None
-        self.q_tensors = None
-        self.bits = bits
-        self.maxq = 2**self.bits - 1
-        self.infeatures = qweight.shape[0] // self.bits * 32
-        self.outfeatures = qweight.shape[1]
+        self.q_tensors = weight
+        self.extra_tensors = _ExtraTensors()
+
+        if isinstance(weight, Exl2Weight):
+            self.infeatures = weight.q_invperm.shape[0]
+            self.outfeatures = weight.q_weight.shape[1]
+        elif isinstance(weight, GPTQWeight):
+            if weight.bits != 4:
+                raise ValueError(
+                    f"Exllamav2 kernel supports only bits=4, requested bits={weight.bits}. Something is wrong in the model initialization."
+                )
+
+            self.infeatures = weight.qweight.shape[0] // weight.bits * 32
+            self.outfeatures = weight.qweight.shape[1]
+
         self.padding = -self.outfeatures % 32
         self.outfeatures = self.outfeatures + self.padding
 
-        self.device = qweight.device
-        self.qweight = qweight
-        self.qzeros = qzeros
-        self.scales = scales
-        self.g_idx = g_idx
+        self.device = weight.device
         self.bias = bias if bias is not None else None
-        self.group_size = groupsize
 
-        global FIXED_BYTES, LAYERS
-        FIXED_BYTES = max(FIXED_BYTES, self.scratch_space_fixed())
+        global LAYERS
         LAYERS.append(self)
 
     def post_init(self, temp_dq):
-        assert self.qweight.device.type == "cuda"
-        assert self.qweight.device.index is not None
-        self.q_tensors = {
-            "qweight": self.qweight,
-            "qzeros": self.qzeros,
-            "scales": self.scales,
-            "g_idx": self.g_idx,
-        }
+        device = self.q_tensors.device
+        assert device.type == "cuda"
+        assert device.index is not None
         temp_dq = temp_dq.get_scratch_slice(self.temp_dq_size())
 
         # We NEED to keep a pointer on Python side, otherwise the garbage collector will mess with us,
         # and `Memory access fault by GPU node-2` will EAT you.
         self.temp_dq = temp_dq
-        self.q_handle = ext_make_q_matrix(self.q_tensors, temp_dq)
+        self.q_handle = ext_make_q_matrix(self.q_tensors, self.extra_tensors, temp_dq)
 
     def forward(self, x, force_cuda=False):
         output = ext_gemm_half_q_half(x, self.q_handle, self.outfeatures, force_cuda)
@@ -201,7 +236,7 @@ class QuantLinear(nn.Module):
     def temp_fwd_size(self, max_input_len, max_batch_size):
         return self.outfeatures * max_input_len * max_batch_size * 4 + 128
 
-    def scratch_space_fixed(self, max_input_len=4096, max_batch_size=16):
+    def scratch_space_fixed(self, max_input_len, max_batch_size):
         return self.temp_dq_size() + self.temp_fwd_size(max_input_len, max_batch_size)
 
 
diff --git a/server/text_generation_server/layers/gptq/exllamav2.py.rej b/server/text_generation_server/layers/gptq/exllamav2.py.rej
deleted file mode 100644
index cde7b73a..00000000
--- a/server/text_generation_server/layers/gptq/exllamav2.py.rej
+++ /dev/null
@@ -1,10 +0,0 @@
-diff a/server/text_generation_server/layers/gptq/exllamav2.py b/server/text_generation_server/layers/gptq/exllamav2.py	(rejected hunks)
-@@ -119,6 +119,8 @@ def ext_make_q_matrix(w: dict, temp_dq, key: str = None):
-                 none_tensor,
-                 temp_dq,
-             )
-+    else:
-+        RuntimeError("Cannot create handle")
- 
- 
- DEVICE = None
diff --git a/server/text_generation_server/layers/gptq/quant_linear.py b/server/text_generation_server/layers/gptq/quant_linear.py
index b52ceb0f..736c357b 100644
--- a/server/text_generation_server/layers/gptq/quant_linear.py
+++ b/server/text_generation_server/layers/gptq/quant_linear.py
@@ -1,356 +1,359 @@
-import math
-import numpy as np
-import torch
-import torch.nn as nn
-from torch.cuda.amp import custom_fwd
-
-import triton
-import triton.language as tl
-from . import custom_autotune
-
-
-# code based https://github.com/fpgaminer/GPTQ-triton
-@custom_autotune.autotune(
-    configs=[
-        triton.Config(
-            {
-                "BLOCK_SIZE_M": 64,
-                "BLOCK_SIZE_N": 256,
-                "BLOCK_SIZE_K": 32,
-                "GROUP_SIZE_M": 8,
-            },
-            num_stages=4,
-            num_warps=4,
-        ),
-        triton.Config(
-            {
-                "BLOCK_SIZE_M": 128,
-                "BLOCK_SIZE_N": 128,
-                "BLOCK_SIZE_K": 32,
-                "GROUP_SIZE_M": 8,
-            },
-            num_stages=4,
-            num_warps=4,
-        ),
-        triton.Config(
-            {
-                "BLOCK_SIZE_M": 64,
-                "BLOCK_SIZE_N": 128,
-                "BLOCK_SIZE_K": 32,
-                "GROUP_SIZE_M": 8,
-            },
-            num_stages=4,
-            num_warps=4,
-        ),
-        triton.Config(
-            {
-                "BLOCK_SIZE_M": 128,
-                "BLOCK_SIZE_N": 32,
-                "BLOCK_SIZE_K": 32,
-                "GROUP_SIZE_M": 8,
-            },
-            num_stages=4,
-            num_warps=4,
-        ),
-        triton.Config(
-            {
-                "BLOCK_SIZE_M": 64,
-                "BLOCK_SIZE_N": 64,
-                "BLOCK_SIZE_K": 32,
-                "GROUP_SIZE_M": 8,
-            },
-            num_stages=4,
-            num_warps=4,
-        ),
-        triton.Config(
-            {
-                "BLOCK_SIZE_M": 64,
-                "BLOCK_SIZE_N": 128,
-                "BLOCK_SIZE_K": 32,
-                "GROUP_SIZE_M": 8,
-            },
-            num_stages=2,
-            num_warps=8,
-        ),
-        triton.Config(
-            {
-                "BLOCK_SIZE_M": 64,
-                "BLOCK_SIZE_N": 64,
-                "BLOCK_SIZE_K": 64,
-                "GROUP_SIZE_M": 8,
-            },
-            num_stages=3,
-            num_warps=8,
-        ),
-        triton.Config(
-            {
-                "BLOCK_SIZE_M": 32,
-                "BLOCK_SIZE_N": 32,
-                "BLOCK_SIZE_K": 128,
-                "GROUP_SIZE_M": 8,
-            },
-            num_stages=2,
-            num_warps=4,
-        ),
-    ],
-    key=["M", "N", "K"],
-    nearest_power_of_two=True,
-    prune_configs_by={
-        "early_config_prune": custom_autotune.matmul248_kernel_config_pruner,
-        "perf_model": None,
-        "top_k": None,
-    },
-)
-@triton.jit
-def matmul_248_kernel(
-    a_ptr,
-    b_ptr,
-    c_ptr,
-    scales_ptr,
-    zeros_ptr,
-    g_ptr,
-    M,
-    N,
-    K,
-    bits,
-    maxq,
-    stride_am,
-    stride_ak,
-    stride_bk,
-    stride_bn,
-    stride_cm,
-    stride_cn,
-    stride_scales,
-    stride_zeros,
-    BLOCK_SIZE_M: tl.constexpr,
-    BLOCK_SIZE_N: tl.constexpr,
-    BLOCK_SIZE_K: tl.constexpr,
-    GROUP_SIZE_M: tl.constexpr,
-):
-    """
-    Compute the matrix multiplication C = A x B.
-    A is of shape (M, K) float16
-    B is of shape (K//8, N) int32
-    C is of shape (M, N) float16
-    scales is of shape (G, N) float16
-    zeros is of shape (G, N) float16
-    g_ptr is of shape (K) int32
-    """
-    infearure_per_bits = 32 // bits
-
-    pid = tl.program_id(axis=0)
-    num_pid_m = tl.cdiv(M, BLOCK_SIZE_M)
-    num_pid_n = tl.cdiv(N, BLOCK_SIZE_N)
-    num_pid_k = tl.cdiv(K, BLOCK_SIZE_K)
-    num_pid_in_group = GROUP_SIZE_M * num_pid_n
-    group_id = pid // num_pid_in_group
-    first_pid_m = group_id * GROUP_SIZE_M
-    group_size_m = min(num_pid_m - first_pid_m, GROUP_SIZE_M)
-    pid_m = first_pid_m + (pid % group_size_m)
-    pid_n = (pid % num_pid_in_group) // group_size_m
-
-    offs_am = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)
-    offs_bn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)
-    offs_k = tl.arange(0, BLOCK_SIZE_K)
-    a_ptrs = a_ptr + (
-        offs_am[:, None] * stride_am + offs_k[None, :] * stride_ak
-    )  # (BLOCK_SIZE_M, BLOCK_SIZE_K)
-    a_mask = offs_am[:, None] < M
-    # b_ptrs is set up such that it repeats elements along the K axis 8 times
-    b_ptrs = b_ptr + (
-        (offs_k[:, None] // infearure_per_bits) * stride_bk
-        + offs_bn[None, :] * stride_bn
-    )  # (BLOCK_SIZE_K, BLOCK_SIZE_N)
-    g_ptrs = g_ptr + offs_k
-    # shifter is used to extract the N bits of each element in the 32-bit word from B
-    scales_ptrs = scales_ptr + offs_bn[None, :]
-    zeros_ptrs = zeros_ptr + (offs_bn[None, :] // infearure_per_bits)
-
-    shifter = (offs_k % infearure_per_bits) * bits
-    zeros_shifter = (offs_bn % infearure_per_bits) * bits
-    accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32)
-
-    for k in range(0, num_pid_k):
-        g_idx = tl.load(g_ptrs)
-
-        # Fetch scales and zeros; these are per-outfeature and thus reused in the inner loop
-        scales = tl.load(
-            scales_ptrs + g_idx[:, None] * stride_scales
-        )  # (BLOCK_SIZE_K, BLOCK_SIZE_N,)
-        zeros = tl.load(
-            zeros_ptrs + g_idx[:, None] * stride_zeros
-        )  # (BLOCK_SIZE_K, BLOCK_SIZE_N,)
-
-        zeros = (zeros >> zeros_shifter[None, :]) & maxq
-        zeros = (zeros + 1) & maxq  # eventually avoid overflow
-
-        a = tl.load(a_ptrs, mask=a_mask, other=0.0)  # (BLOCK_SIZE_M, BLOCK_SIZE_K)
-        b = tl.load(b_ptrs)  # (BLOCK_SIZE_K, BLOCK_SIZE_N), but repeated
-
-        # Now we need to unpack b (which is N-bit values) into 32-bit values
-        b = (b >> shifter[:, None]) & maxq  # Extract the N-bit values
-        b = (b - zeros) * scales  # Scale and shift
-
-        accumulator += tl.dot(a, b)
-        a_ptrs += BLOCK_SIZE_K
-        b_ptrs += (BLOCK_SIZE_K // infearure_per_bits) * stride_bk
-        g_ptrs += BLOCK_SIZE_K
-
-    c_ptrs = c_ptr + stride_cm * offs_am[:, None] + stride_cn * offs_bn[None, :]
-    c_mask = (offs_am[:, None] < M) & (offs_bn[None, :] < N)
-    tl.store(c_ptrs, accumulator, mask=c_mask)
-
-
-def matmul248(input, qweight, scales, qzeros, g_idx, bits, maxq):
-    with torch.cuda.device(input.device):
-        output = torch.empty(
-            (input.shape[0], qweight.shape[1]), device=input.device, dtype=torch.float16
-        )
-        grid = lambda META: (
-            triton.cdiv(input.shape[0], META["BLOCK_SIZE_M"])
-            * triton.cdiv(qweight.shape[1], META["BLOCK_SIZE_N"]),
-        )
-        matmul_248_kernel[grid](
-            input,
-            qweight,
-            output,
-            scales,
-            qzeros,
-            g_idx,
-            input.shape[0],
-            qweight.shape[1],
-            input.shape[1],
-            bits,
-            maxq,
-            input.stride(0),
-            input.stride(1),
-            qweight.stride(0),
-            qweight.stride(1),
-            output.stride(0),
-            output.stride(1),
-            scales.stride(0),
-            qzeros.stride(0),
-        )
-        return output
-
-
-class QuantLinearFunction(torch.autograd.Function):
-    @staticmethod
-    @custom_fwd(cast_inputs=torch.float16)
-    def forward(ctx, input, qweight, scales, qzeros, g_idx, bits, maxq):
-        output = matmul248(input, qweight, scales, qzeros, g_idx, bits, maxq)
-        return output
-
-
-class QuantLinear(nn.Module):
-    def __init__(self, qweight, qzeros, scales, g_idx, bias, bits, groupsize):
-        super().__init__()
-        self.register_buffer("qweight", qweight)
-        self.register_buffer("qzeros", qzeros)
-        self.register_buffer("scales", scales)
-        self.register_buffer("g_idx", g_idx)
-        if bias is not None:
-            self.register_buffer("bias", bias)
-        else:
-            self.bias = None
-        if bits not in [2, 4, 8]:
-            raise NotImplementedError("Only 2,4,8 bits are supported.")
-        self.bits = bits
-        self.maxq = 2**self.bits - 1
-        self.groupsize = groupsize
-
-        self.outfeatures = qweight.shape[1]
-        self.infeatures = qweight.shape[0] * 32 // bits
-
-    @classmethod
-    def new(cls, bits, groupsize, infeatures, outfeatures, bias):
-        if bits not in [2, 4, 8]:
-            raise NotImplementedError("Only 2,4,8 bits are supported.")
-
-        qweight = torch.zeros((infeatures // 32 * bits, outfeatures), dtype=torch.int32)
-        qzeros = torch.zeros(
-            (math.ceil(infeatures / groupsize), outfeatures // 32 * bits),
-            dtype=torch.int32,
-        )
-        scales = torch.zeros(
-            (math.ceil(infeatures / groupsize), outfeatures), dtype=torch.float16
-        )
-        g_idx = torch.tensor(
-            [i // groupsize for i in range(infeatures)], dtype=torch.int32
-        )
-        if bias:
-            bias = torch.zeros((outfeatures), dtype=torch.float16)
-        else:
-            bias = None
-        return cls(qweight, qzeros, scales, g_idx, bias, bits, groupsize)
-
-    def pack(self, linear, scales, zeros, g_idx=None):
-        self.g_idx = g_idx.clone() if g_idx is not None else self.g_idx
-
-        scales = scales.t().contiguous()
-        zeros = zeros.t().contiguous()
-        scale_zeros = zeros * scales
-        self.scales = scales.clone().half()
-        if linear.bias is not None:
-            self.bias = linear.bias.clone().half()
-
-        intweight = []
-        for idx in range(self.infeatures):
-            intweight.append(
-                torch.round(
-                    (linear.weight.data[:, idx] + scale_zeros[self.g_idx[idx]])
-                    / self.scales[self.g_idx[idx]]
-                ).to(torch.int)[:, None]
-            )
-        intweight = torch.cat(intweight, dim=1)
-        intweight = intweight.t().contiguous()
-        intweight = intweight.numpy().astype(np.uint32)
-        qweight = np.zeros(
-            (intweight.shape[0] // 32 * self.bits, intweight.shape[1]), dtype=np.uint32
-        )
-        i = 0
-        row = 0
-        while row < qweight.shape[0]:
-            if self.bits in [2, 4, 8]:
-                for j in range(i, i + (32 // self.bits)):
-                    qweight[row] |= intweight[j] << (self.bits * (j - i))
-                i += 32 // self.bits
-                row += 1
-            else:
-                raise NotImplementedError("Only 2,4,8 bits are supported.")
-
-        qweight = qweight.astype(np.int32)
-        self.qweight = torch.from_numpy(qweight)
-
-        zeros -= 1
-        zeros = zeros.numpy().astype(np.uint32)
-        qzeros = np.zeros(
-            (zeros.shape[0], zeros.shape[1] // 32 * self.bits), dtype=np.uint32
-        )
-        i = 0
-        col = 0
-        while col < qzeros.shape[1]:
-            if self.bits in [2, 4, 8]:
-                for j in range(i, i + (32 // self.bits)):
-                    qzeros[:, col] |= zeros[:, j] << (self.bits * (j - i))
-                i += 32 // self.bits
-                col += 1
-            else:
-                raise NotImplementedError("Only 2,4,8 bits are supported.")
-
-        qzeros = qzeros.astype(np.int32)
-        self.qzeros = torch.from_numpy(qzeros)
-
-    def forward(self, x):
-        out_shape = x.shape[:-1] + (self.outfeatures,)
-        out = QuantLinearFunction.apply(
-            x.reshape(-1, x.shape[-1]),
-            self.qweight,
-            self.scales,
-            self.qzeros,
-            self.g_idx,
-            self.bits,
-            self.maxq,
-        )
-        out = out + self.bias if self.bias is not None else out
-        return out.reshape(out_shape)
+import math
+import numpy as np
+import torch
+import torch.nn as nn
+from torch.cuda.amp import custom_fwd
+
+import triton
+import triton.language as tl
+from . import custom_autotune
+
+
+# code based https://github.com/fpgaminer/GPTQ-triton
+@custom_autotune.autotune(
+    configs=[
+        triton.Config(
+            {
+                "BLOCK_SIZE_M": 64,
+                "BLOCK_SIZE_N": 256,
+                "BLOCK_SIZE_K": 32,
+                "GROUP_SIZE_M": 8,
+            },
+            num_stages=4,
+            num_warps=4,
+        ),
+        triton.Config(
+            {
+                "BLOCK_SIZE_M": 128,
+                "BLOCK_SIZE_N": 128,
+                "BLOCK_SIZE_K": 32,
+                "GROUP_SIZE_M": 8,
+            },
+            num_stages=4,
+            num_warps=4,
+        ),
+        triton.Config(
+            {
+                "BLOCK_SIZE_M": 64,
+                "BLOCK_SIZE_N": 128,
+                "BLOCK_SIZE_K": 32,
+                "GROUP_SIZE_M": 8,
+            },
+            num_stages=4,
+            num_warps=4,
+        ),
+        triton.Config(
+            {
+                "BLOCK_SIZE_M": 128,
+                "BLOCK_SIZE_N": 32,
+                "BLOCK_SIZE_K": 32,
+                "GROUP_SIZE_M": 8,
+            },
+            num_stages=4,
+            num_warps=4,
+        ),
+        triton.Config(
+            {
+                "BLOCK_SIZE_M": 64,
+                "BLOCK_SIZE_N": 64,
+                "BLOCK_SIZE_K": 32,
+                "GROUP_SIZE_M": 8,
+            },
+            num_stages=4,
+            num_warps=4,
+        ),
+        triton.Config(
+            {
+                "BLOCK_SIZE_M": 64,
+                "BLOCK_SIZE_N": 128,
+                "BLOCK_SIZE_K": 32,
+                "GROUP_SIZE_M": 8,
+            },
+            num_stages=2,
+            num_warps=8,
+        ),
+        triton.Config(
+            {
+                "BLOCK_SIZE_M": 64,
+                "BLOCK_SIZE_N": 64,
+                "BLOCK_SIZE_K": 64,
+                "GROUP_SIZE_M": 8,
+            },
+            num_stages=3,
+            num_warps=8,
+        ),
+        triton.Config(
+            {
+                "BLOCK_SIZE_M": 32,
+                "BLOCK_SIZE_N": 32,
+                "BLOCK_SIZE_K": 128,
+                "GROUP_SIZE_M": 8,
+            },
+            num_stages=2,
+            num_warps=4,
+        ),
+    ],
+    key=["M", "N", "K"],
+    nearest_power_of_two=True,
+    prune_configs_by={
+        "early_config_prune": custom_autotune.matmul248_kernel_config_pruner,
+        "perf_model": None,
+        "top_k": None,
+    },
+)
+@triton.jit
+def matmul_248_kernel(
+    a_ptr,
+    b_ptr,
+    c_ptr,
+    scales_ptr,
+    zeros_ptr,
+    g_ptr,
+    M,
+    N,
+    K,
+    bits,
+    maxq,
+    stride_am,
+    stride_ak,
+    stride_bk,
+    stride_bn,
+    stride_cm,
+    stride_cn,
+    stride_scales,
+    stride_zeros,
+    BLOCK_SIZE_M: tl.constexpr,
+    BLOCK_SIZE_N: tl.constexpr,
+    BLOCK_SIZE_K: tl.constexpr,
+    GROUP_SIZE_M: tl.constexpr,
+):
+    """
+    Compute the matrix multiplication C = A x B.
+    A is of shape (M, K) float16
+    B is of shape (K//8, N) int32
+    C is of shape (M, N) float16
+    scales is of shape (G, N) float16
+    zeros is of shape (G, N) float16
+    g_ptr is of shape (K) int32
+    """
+    infearure_per_bits = 32 // bits
+
+    pid = tl.program_id(axis=0)
+    num_pid_m = tl.cdiv(M, BLOCK_SIZE_M)
+    num_pid_n = tl.cdiv(N, BLOCK_SIZE_N)
+    num_pid_k = tl.cdiv(K, BLOCK_SIZE_K)
+    num_pid_in_group = GROUP_SIZE_M * num_pid_n
+    group_id = pid // num_pid_in_group
+    first_pid_m = group_id * GROUP_SIZE_M
+    group_size_m = min(num_pid_m - first_pid_m, GROUP_SIZE_M)
+    pid_m = first_pid_m + (pid % group_size_m)
+    pid_n = (pid % num_pid_in_group) // group_size_m
+
+    offs_am = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)
+    offs_bn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)
+    offs_k = tl.arange(0, BLOCK_SIZE_K)
+    a_ptrs = a_ptr + (
+        offs_am[:, None] * stride_am + offs_k[None, :] * stride_ak
+    )  # (BLOCK_SIZE_M, BLOCK_SIZE_K)
+    a_mask = offs_am[:, None] < M
+    # b_ptrs is set up such that it repeats elements along the K axis 8 times
+    b_ptrs = b_ptr + (
+        (offs_k[:, None] // infearure_per_bits) * stride_bk
+        + offs_bn[None, :] * stride_bn
+    )  # (BLOCK_SIZE_K, BLOCK_SIZE_N)
+    g_ptrs = g_ptr + offs_k
+    # shifter is used to extract the N bits of each element in the 32-bit word from B
+    scales_ptrs = scales_ptr + offs_bn[None, :]
+    zeros_ptrs = zeros_ptr + (offs_bn[None, :] // infearure_per_bits)
+
+    shifter = (offs_k % infearure_per_bits) * bits
+    zeros_shifter = (offs_bn % infearure_per_bits) * bits
+    accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32)
+
+    for k in range(0, num_pid_k):
+        g_idx = tl.load(g_ptrs)
+
+        # Fetch scales and zeros; these are per-outfeature and thus reused in the inner loop
+        scales = tl.load(
+            scales_ptrs + g_idx[:, None] * stride_scales
+        )  # (BLOCK_SIZE_K, BLOCK_SIZE_N,)
+        zeros = tl.load(
+            zeros_ptrs + g_idx[:, None] * stride_zeros
+        )  # (BLOCK_SIZE_K, BLOCK_SIZE_N,)
+
+        zeros = (zeros >> zeros_shifter[None, :]) & maxq
+        zeros = (zeros + 1) & maxq  # eventually avoid overflow
+
+        a = tl.load(a_ptrs, mask=a_mask, other=0.0)  # (BLOCK_SIZE_M, BLOCK_SIZE_K)
+        b = tl.load(b_ptrs)  # (BLOCK_SIZE_K, BLOCK_SIZE_N), but repeated
+
+        # Now we need to unpack b (which is N-bit values) into 32-bit values
+        b = (b >> shifter[:, None]) & maxq  # Extract the N-bit values
+        b = (b - zeros) * scales  # Scale and shift
+
+        accumulator += tl.dot(a, b)
+        a_ptrs += BLOCK_SIZE_K
+        b_ptrs += (BLOCK_SIZE_K // infearure_per_bits) * stride_bk
+        g_ptrs += BLOCK_SIZE_K
+
+    c_ptrs = c_ptr + stride_cm * offs_am[:, None] + stride_cn * offs_bn[None, :]
+    c_mask = (offs_am[:, None] < M) & (offs_bn[None, :] < N)
+    tl.store(c_ptrs, accumulator, mask=c_mask)
+
+
+def matmul248(input, qweight, scales, qzeros, g_idx, bits, maxq):
+    with torch.cuda.device(input.device):
+        output = torch.empty(
+            (input.shape[0], qweight.shape[1]), device=input.device, dtype=torch.float16
+        )
+
+        def grid(META):
+            return (
+                triton.cdiv(input.shape[0], META["BLOCK_SIZE_M"])
+                * triton.cdiv(qweight.shape[1], META["BLOCK_SIZE_N"]),
+            )
+
+        matmul_248_kernel[grid](
+            input,
+            qweight,
+            output,
+            scales,
+            qzeros,
+            g_idx,
+            input.shape[0],
+            qweight.shape[1],
+            input.shape[1],
+            bits,
+            maxq,
+            input.stride(0),
+            input.stride(1),
+            qweight.stride(0),
+            qweight.stride(1),
+            output.stride(0),
+            output.stride(1),
+            scales.stride(0),
+            qzeros.stride(0),
+        )
+        return output
+
+
+class QuantLinearFunction(torch.autograd.Function):
+    @staticmethod
+    @custom_fwd(cast_inputs=torch.float16)
+    def forward(ctx, input, qweight, scales, qzeros, g_idx, bits, maxq):
+        output = matmul248(input, qweight, scales, qzeros, g_idx, bits, maxq)
+        return output
+
+
+class QuantLinear(nn.Module):
+    def __init__(self, qweight, qzeros, scales, g_idx, bias, bits, groupsize):
+        super().__init__()
+        self.register_buffer("qweight", qweight)
+        self.register_buffer("qzeros", qzeros)
+        self.register_buffer("scales", scales)
+        self.register_buffer("g_idx", g_idx)
+        if bias is not None:
+            self.register_buffer("bias", bias)
+        else:
+            self.bias = None
+        if bits not in [2, 4, 8]:
+            raise NotImplementedError("Only 2,4,8 bits are supported.")
+        self.bits = bits
+        self.maxq = 2**self.bits - 1
+        self.groupsize = groupsize
+
+        self.outfeatures = qweight.shape[1]
+        self.infeatures = qweight.shape[0] * 32 // bits
+
+    @classmethod
+    def new(cls, bits, groupsize, infeatures, outfeatures, bias):
+        if bits not in [2, 4, 8]:
+            raise NotImplementedError("Only 2,4,8 bits are supported.")
+
+        qweight = torch.zeros((infeatures // 32 * bits, outfeatures), dtype=torch.int32)
+        qzeros = torch.zeros(
+            (math.ceil(infeatures / groupsize), outfeatures // 32 * bits),
+            dtype=torch.int32,
+        )
+        scales = torch.zeros(
+            (math.ceil(infeatures / groupsize), outfeatures), dtype=torch.float16
+        )
+        g_idx = torch.tensor(
+            [i // groupsize for i in range(infeatures)], dtype=torch.int32
+        )
+        if bias:
+            bias = torch.zeros((outfeatures), dtype=torch.float16)
+        else:
+            bias = None
+        return cls(qweight, qzeros, scales, g_idx, bias, bits, groupsize)
+
+    def pack(self, linear, scales, zeros, g_idx=None):
+        self.g_idx = g_idx.clone() if g_idx is not None else self.g_idx
+
+        scales = scales.t().contiguous()
+        zeros = zeros.t().contiguous()
+        scale_zeros = zeros * scales
+        self.scales = scales.clone().half()
+        if linear.bias is not None:
+            self.bias = linear.bias.clone().half()
+
+        intweight = []
+        for idx in range(self.infeatures):
+            intweight.append(
+                torch.round(
+                    (linear.weight.data[:, idx] + scale_zeros[self.g_idx[idx]])
+                    / self.scales[self.g_idx[idx]]
+                ).to(torch.int)[:, None]
+            )
+        intweight = torch.cat(intweight, dim=1)
+        intweight = intweight.t().contiguous()
+        intweight = intweight.numpy().astype(np.uint32)
+        qweight = np.zeros(
+            (intweight.shape[0] // 32 * self.bits, intweight.shape[1]), dtype=np.uint32
+        )
+        i = 0
+        row = 0
+        while row < qweight.shape[0]:
+            if self.bits in [2, 4, 8]:
+                for j in range(i, i + (32 // self.bits)):
+                    qweight[row] |= intweight[j] << (self.bits * (j - i))
+                i += 32 // self.bits
+                row += 1
+            else:
+                raise NotImplementedError("Only 2,4,8 bits are supported.")
+
+        qweight = qweight.astype(np.int32)
+        self.qweight = torch.from_numpy(qweight)
+
+        zeros -= 1
+        zeros = zeros.numpy().astype(np.uint32)
+        qzeros = np.zeros(
+            (zeros.shape[0], zeros.shape[1] // 32 * self.bits), dtype=np.uint32
+        )
+        i = 0
+        col = 0
+        while col < qzeros.shape[1]:
+            if self.bits in [2, 4, 8]:
+                for j in range(i, i + (32 // self.bits)):
+                    qzeros[:, col] |= zeros[:, j] << (self.bits * (j - i))
+                i += 32 // self.bits
+                col += 1
+            else:
+                raise NotImplementedError("Only 2,4,8 bits are supported.")
+
+        qzeros = qzeros.astype(np.int32)
+        self.qzeros = torch.from_numpy(qzeros)
+
+    def forward(self, x):
+        out_shape = x.shape[:-1] + (self.outfeatures,)
+        out = QuantLinearFunction.apply(
+            x.reshape(-1, x.shape[-1]),
+            self.qweight,
+            self.scales,
+            self.qzeros,
+            self.g_idx,
+            self.bits,
+            self.maxq,
+        )
+        out = out + self.bias if self.bias is not None else out
+        return out.reshape(out_shape)
diff --git a/server/text_generation_server/layers/gptq/quantize.py b/server/text_generation_server/layers/gptq/quantize.py
index ca113d8f..b0086ea0 100644
--- a/server/text_generation_server/layers/gptq/quantize.py
+++ b/server/text_generation_server/layers/gptq/quantize.py
@@ -12,9 +12,12 @@ from huggingface_hub import HfApi
 from accelerate import init_empty_weights
 from text_generation_server.utils import initialize_torch_distributed, Weights
 from text_generation_server.utils.hub import weight_files
-from text_generation_server.utils.gptq.quant_linear import QuantLinear
+from text_generation_server.layers.gptq.quant_linear import QuantLinear
 from loguru import logger
 from typing import Optional
+from text_generation_server.layers.gptq.utils import torch_snr_error
+
+from text_generation_server.utils.weights import DefaultWeightsLoader, UnquantizedWeight
 
 DEV = torch.device("cuda:0")
 
@@ -370,7 +373,7 @@ def get_wikitext2(nsamples, seed, seqlen, model_id, trust_remote_code):
         tokenizer = AutoTokenizer.from_pretrained(
             model_id, use_fast=False, trust_remote_code=trust_remote_code
         )
-    except:
+    except Exception:
         tokenizer = AutoTokenizer.from_pretrained(
             model_id, use_fast=True, trust_remote_code=trust_remote_code
         )
@@ -402,7 +405,7 @@ def get_ptb(nsamples, seed, seqlen, model_id, trust_remote_code):
         tokenizer = AutoTokenizer.from_pretrained(
             model_id, use_fast=False, trust_remote_code=trust_remote_code
         )
-    except:
+    except Exception:
         tokenizer = AutoTokenizer.from_pretrained(
             model_id, use_fast=True, trust_remote_code=trust_remote_code
         )
@@ -446,7 +449,7 @@ def get_c4(nsamples, seed, seqlen, model_id, trust_remote_code):
         tokenizer = AutoTokenizer.from_pretrained(
             model_id, use_fast=False, trust_remote_code=trust_remote_code
         )
-    except:
+    except Exception:
         tokenizer = AutoTokenizer.from_pretrained(
             model_id, use_fast=True, trust_remote_code=trust_remote_code
         )
@@ -502,7 +505,7 @@ def get_ptb_new(nsamples, seed, seqlen, model_id, trust_remote_code):
         tokenizer = AutoTokenizer.from_pretrained(
             model_id, use_fast=False, trust_remote_code=trust_remote_code
         )
-    except:
+    except Exception:
         tokenizer = AutoTokenizer.from_pretrained(
             model_id, use_fast=True, trust_remote_code=trust_remote_code
         )
@@ -544,7 +547,7 @@ def get_c4_new(nsamples, seed, seqlen, model_id, trust_remote_code):
         tokenizer = AutoTokenizer.from_pretrained(
             model_id, use_fast=False, trust_remote_code=trust_remote_code
         )
-    except:
+    except Exception:
         tokenizer = AutoTokenizer.from_pretrained(
             model_id, use_fast=True, trust_remote_code=trust_remote_code
         )
@@ -698,6 +701,8 @@ def sequential(
                 pass
 
             def add_batch(name):
+                nonlocal gptq
+
                 def tmp(_, inp, out):
                     gptq[name].add_batch(inp[0].data, out.data)
 
@@ -869,6 +874,7 @@ def quantize(
     upload_to_model_id: Optional[str],
     percdamp: float,
     act_order: bool,
+    sym: bool,
 ):
     print("loading model")
     config = AutoConfig.from_pretrained(
@@ -891,6 +897,7 @@ def quantize(
         dtype=torch.float16,
         process_group=process_group,
         aliases={"embed_tokens.weight": ["lm_head.weight"]},
+        weights_loader=DefaultWeightsLoader(UnquantizedWeight),
     )
     hooks = []
     for name, module in model.named_modules():
@@ -943,6 +950,7 @@ def quantize(
         percdamp=percdamp,
         act_order=act_order,
         hooks=hooks,
+        sym=sym,
     )
     print(time.time() - tick)
 
@@ -952,8 +960,6 @@ def quantize(
 
     state_dict = model.state_dict()
     state_dict = {k: v.cpu().contiguous() for k, v in state_dict.items()}
-    state_dict["gptq_bits"] = torch.LongTensor([bits])
-    state_dict["gptq_groupsize"] = torch.LongTensor([groupsize])
 
     max_shard_size = "10GB"
     shards, index = shard_checkpoint(
@@ -985,6 +991,15 @@ def quantize(
             f"index located at {save_index_file}."
         )
     config = AutoConfig.from_pretrained(model_id, trust_remote_code=trust_remote_code)
+    config.quantization_config = {
+        "bits": bits,
+        "group_size": groupsize,
+        "damp_percent": percdamp,
+        "desc_act": act_order,
+        "static_groups": False,
+        "sym": sym,
+        "quant_method": "gptq",
+    }
     config.save_pretrained(output_dir)
     logger.info("Saved config")
     logger.info("Saving tokenizer")
diff --git a/server/text_generation_server/layers/gptq/utils.py b/server/text_generation_server/layers/gptq/utils.py
new file mode 100644
index 00000000..cbc0f391
--- /dev/null
+++ b/server/text_generation_server/layers/gptq/utils.py
@@ -0,0 +1,56 @@
+import torch
+
+
+# copied from https://github.com/openppl-public/ppq/blob/master/ppq/quantization/measure/norm.py
+def torch_snr_error(
+    y_pred: torch.Tensor, y_real: torch.Tensor, reduction: str = "mean"
+) -> torch.Tensor:
+    """
+    Compute SNR between y_pred(tensor) and y_real(tensor)
+
+    SNR can be calcualted as following equation:
+
+        SNR(pred, real) = (pred - real) ^ 2 / (real) ^ 2
+
+    if x and y are matrixs, SNR error over matrix should be the mean value of SNR error over all elements.
+
+        SNR(pred, real) = mean((pred - real) ^ 2 / (real) ^ 2)
+
+    Args:
+        y_pred (torch.Tensor): _description_
+        y_real (torch.Tensor): _description_
+        reduction (str, optional): _description_. Defaults to 'mean'.
+
+    Raises:
+        ValueError: _description_
+        ValueError: _description_
+
+    Returns:
+        torch.Tensor: _description_
+    """
+    if y_pred.shape != y_real.shape:
+        raise ValueError(
+            f"Can not compute snr loss for tensors with different shape. "
+            f"({y_pred.shape} and {y_real.shape})"
+        )
+    reduction = str(reduction).lower()
+
+    if y_pred.ndim == 1:
+        y_pred = y_pred.unsqueeze(0)
+        y_real = y_real.unsqueeze(0)
+
+    y_pred = y_pred.flatten(start_dim=1)
+    y_real = y_real.flatten(start_dim=1)
+
+    noise_power = torch.pow(y_pred - y_real, 2).sum(dim=-1)
+    signal_power = torch.pow(y_real, 2).sum(dim=-1)
+    snr = (noise_power) / (signal_power + 1e-7)
+
+    if reduction == "mean":
+        return torch.mean(snr)
+    elif reduction == "sum":
+        return torch.sum(snr)
+    elif reduction == "none":
+        return snr
+    else:
+        raise ValueError("Unsupported reduction method.")
diff --git a/server/text_generation_server/layers/layernorm.py b/server/text_generation_server/layers/layernorm.py
index c4aa6c7d..ce5289f9 100644
--- a/server/text_generation_server/layers/layernorm.py
+++ b/server/text_generation_server/layers/layernorm.py
@@ -82,18 +82,20 @@ elif SYSTEM == "rocm":
 
             return super().forward(hidden_states), residual
 
-elif SYSTEM == "xpu":
+elif SYSTEM == "ipex":
     import intel_extension_for_pytorch as ipex
 
     class FastLayerNorm(nn.LayerNorm):
         def forward(self, hidden_states, residual=None):
-            res_out = hidden_states
             out = ipex.llm.functional.add_layer_norm(
-                residual, hidden_states, self.weight, self.bias, self.eps, True
+                residual,
+                hidden_states,
+                self.weight,
+                self.bias,
+                self.eps,
+                residual is not None,
             )
-            if residual is not None:
-                res_out = residual
-            return out, res_out
+            return out, residual if residual is not None else hidden_states
 
 
 class FastRMSNorm(nn.Module):
@@ -109,19 +111,16 @@ class FastRMSNorm(nn.Module):
         return cls(weight, eps)
 
     def forward(self, hidden_states, residual=None):
-        if SYSTEM == "xpu":
-            residual_out = hidden_states
+        if SYSTEM == "ipex":
             out = ipex.llm.functional.add_rms_norm(
                 residual,
                 hidden_states,
                 self.weight,
                 None,
                 self.variance_epsilon,
-                True,
+                residual is not None,
             )
-            if residual is not None:
-                residual_out = residual
-            return out, residual_out
+            return out, residual if residual is not None else hidden_states
         elif hidden_states.shape[-1] > 8192:
             if residual is not None:
                 hidden_states += residual
diff --git a/server/text_generation_server/layers/linear.py b/server/text_generation_server/layers/linear.py
index 5bd6aa95..08306d57 100644
--- a/server/text_generation_server/layers/linear.py
+++ b/server/text_generation_server/layers/linear.py
@@ -1,12 +1,21 @@
 import torch
-from torch.nn import functional as F
 from text_generation_server.utils.import_utils import SYSTEM
+from torch.nn import functional as F
+import os
 
 if SYSTEM == "rocm":
-    try:
-        from vllm import _custom_C
-    except Exception as e:
-        raise ImportError(f"Could not load `vllm._custom_C`. Full error: {e}")
+    ROCM_USE_SKINNY_GEMM = os.getenv("ROCM_USE_SKINNY_GEMM", "True").lower() in (
+        "true",
+        "1",
+    )
+
+    if ROCM_USE_SKINNY_GEMM:
+        try:
+            from vllm import _custom_C
+        except Exception as e:
+            raise ImportError(
+                f"Could not load `vllm._custom_C` for ROCm skinny gemm. Full error: {e}"
+            )
 
 
 class FastLinear(torch.nn.Module):
@@ -48,6 +57,14 @@ class FastLinearROCm(torch.nn.Module):
         else:
             self.bias = None
 
+        self.cu_count = torch.cuda.get_device_properties(
+            device="cuda"
+        ).multi_processor_count
+        self.use_skinny_gemm = (
+            ROCM_USE_SKINNY_GEMM
+            and "gfx1" not in torch.cuda.get_device_properties("cuda").gcnArchName
+        )
+
     @classmethod
     def load(cls, config, prefix: str, weights, bias: bool):
         weight = weights.get_tensor(f"{prefix}.weight")
@@ -61,7 +78,11 @@ class FastLinearROCm(torch.nn.Module):
         weight = self.weight
         bias = self.bias
 
-        if SYSTEM == "rocm" and inp.numel() // inp.shape[-1] == 1:
+        if (
+            self.use_skinny_gemm
+            and inp.dtype == torch.float16
+            and inp.shape[-1] % 8 == 0
+        ):
             batched = False
             inp_shape = inp.shape
 
@@ -69,13 +90,16 @@ class FastLinearROCm(torch.nn.Module):
                 inp = inp.view(-1, inp_shape[-1])
                 batched = True
 
-            m, k = weight.shape[0], inp_shape[1]
-            out = torch.empty(
-                inp_shape[0], weight.shape[0], dtype=inp.dtype, device="cuda"
-            )
-            if (k == 8192 and (m == 1280 or m == 7168)) or (k == 3584 and m == 8192):
-                _custom_C.LLMM1(weight, inp, out, 8)
-            elif k <= 8192 and k % 8 == 0 and m % 4 == 0:
+            m, n, k = weight.shape[0], inp_shape[0], inp_shape[1]
+            if m > 8 and n <= 4:
+                out = torch.empty(
+                    inp_shape[0], weight.shape[0], dtype=inp.dtype, device=weight.device
+                )
+                _custom_C.wvSpltK(weight, inp, out, n, self.cu_count)
+            elif m % 4 == 0 and n == 1 and k <= 8192:
+                out = torch.empty(
+                    inp_shape[0], weight.shape[0], dtype=inp.dtype, device=weight.device
+                )
                 _custom_C.LLMM1(weight, inp, out, 4)
             else:
                 out = F.linear(inp, weight)
@@ -89,128 +113,14 @@ class FastLinearROCm(torch.nn.Module):
         return F.linear(inp, self.weight, self.bias)
 
 
-def get_linear(weight, bias, quantize):
-    if quantize is None:
+def get_linear(weight, bias):
+    # Weights that are loaded through methods that are not
+    # quantization-aware are still bare tensors. We may want
+    # to change this in the future.
+    if isinstance(weight, torch.Tensor):
         if SYSTEM == "rocm":
-            linear = FastLinearROCm(weight, bias)
+            return FastLinearROCm(weight, bias)
         else:
-            linear = FastLinear(weight, bias)
-    elif quantize == "eetq":
-        try:
-            from text_generation_server.layers.eetq import EETQLinear
+            return FastLinear(weight, bias)
 
-            linear = EETQLinear(weight, bias)
-        except ImportError:
-            raise ImportError(
-                "Please install EETQ from https://github.com/NetEase-FuXi/EETQ"
-            )
-    elif quantize == "fp8":
-        from text_generation_server.layers.fp8 import Fp8Linear
-
-        linear = Fp8Linear(weight, bias)
-    elif quantize == "bitsandbytes":
-        try:
-            from text_generation_server.layers.bnb import (
-                warn_deprecate_bnb,
-                Linear8bitLt,
-            )
-        except ImportError:
-            raise NotImplementedError(
-                f"Bitsandbytes is missing install it with `pip install bitsandbytes`."
-            )
-        warn_deprecate_bnb()
-        linear = Linear8bitLt(
-            weight,
-            bias,
-            has_fp16_weights=False,
-            threshold=6.0,
-        )
-        if bias is not None:
-            linear.bias = nn.Parameter(bias)
-    elif quantize == "bitsandbytes-fp4":
-        try:
-            from text_generation_server.layers.bnb import Linear4bit
-        except ImportError:
-            raise NotImplementedError(
-                f"Bitsandbytes is missing install it with `pip install bitsandbytes`."
-            )
-        linear = Linear4bit(
-            weight,
-            bias,
-            quant_type="fp4",
-        )
-    elif quantize == "bitsandbytes-nf4":
-        try:
-            from text_generation_server.layers.bnb import Linear4bit
-        except ImportError:
-            raise NotImplementedError(
-                f"Bitsandbytes is missing install it with `pip install bitsandbytes`."
-            )
-        linear = Linear4bit(
-            weight,
-            bias,
-            quant_type="nf4",
-        )
-    elif quantize == "gptq":
-        try:
-            qweight, qzeros, scales, g_idx, bits, groupsize, use_exllama = weight
-        except Exception:
-            raise NotImplementedError(
-                f"The passed weight is not `gptq` compatible, loader needs to be updated."
-            )
-
-        if use_exllama:
-            try:
-                from text_generation_server.layers.gptq import (
-                    ExllamaQuantLinear,
-                )
-            except ImportError:
-                raise NotImplementedError(
-                    f"Exllama gptq kernels are not installed. Install them `cd server/exllama_kernels && python setup.py install && cd ../exllamav2_kernels && python setup.py install`"
-                )
-
-            linear = ExllamaQuantLinear(
-                qweight, qzeros, scales, g_idx, bias, bits, groupsize
-            )
-        else:
-            from text_generation_server.layers.gptq.quant_linear import QuantLinear
-
-            linear = QuantLinear(
-                qweight,
-                qzeros,
-                scales,
-                g_idx,
-                bias,
-                bits,
-                groupsize,
-            )
-    elif quantize == "awq":
-        try:
-            qweight, qzeros, scales, _, bits, groupsize, _ = weight
-        except Exception:
-            raise NotImplementedError(
-                f"The passed weight is not `awq` compatible, loader needs to be updated."
-            )
-        if SYSTEM == "rocm":
-            raise NotImplementedError(
-                "AWQ GEMM kernel can't be used on ROCm systems, please use `--quantize gptq` instead "
-                "to use Exllama/GPTQ kernels for AWQ inference."
-            )
-        try:
-            from text_generation_server.layers.awq.quantize.qmodule import WQLinear
-
-            linear = WQLinear(
-                w_bit=bits,
-                group_size=groupsize,
-                qweight=qweight,
-                qzeros=qzeros,
-                scales=scales,
-                bias=bias is not None,
-            )
-        except ImportError:
-            raise NotImplementedError(
-                "You do not seem to have awq installed, either install it (cd server &&  make install-awq), or try using GPTQ `---quantize gptq` a conversion AWQ->GPTQ will happen on the fly"
-            )
-    else:
-        raise NotImplementedError(f"Quantization `{quantize}` is not implemented yet.")
-    return linear
+    return weight.get_linear(bias)
diff --git a/server/text_generation_server/layers/lora.py b/server/text_generation_server/layers/lora.py
new file mode 100644
index 00000000..a4537b55
--- /dev/null
+++ b/server/text_generation_server/layers/lora.py
@@ -0,0 +1,279 @@
+from typing import TYPE_CHECKING, Optional, List
+
+import torch
+import torch.distributed
+from torch import nn
+from torch.distributed import ProcessGroup
+
+from text_generation_server.utils.sgmv import (
+    add_lora_a_bgmv,
+    add_lora_b_bgmv,
+    has_sgmv,
+    lora_a_sgmv_cutlass,
+    lora_b_sgmv_cutlass,
+    orient_for_rank,
+)
+
+if TYPE_CHECKING:
+    from text_generation_server.adapters import AdapterBatchData
+    from text_generation_server.adapters.lora import BatchLoraWeights
+
+
+class LoraLinear(nn.Module):
+    def __init__(
+        self, base_layer: nn.Module, layer_id: int, process_group: ProcessGroup
+    ):
+        super().__init__()
+        self.base_layer = base_layer
+        self.layer_id = layer_id
+        self.process_group = process_group
+
+    def forward_layer_type(
+        self,
+        result: torch.Tensor,
+        input: torch.Tensor,
+        adapter_data: "AdapterBatchData",
+        layer_type: str,
+        start_idx: int,
+        end_idx: int,
+    ) -> torch.Tensor:
+        if adapter_data is None:
+            return result
+        data: Optional["BatchLoraWeights"] = adapter_data.data.get(layer_type)
+
+        if has_sgmv() and data is not None and data.can_vectorize(self.process_group):
+            # In tensor-parallel configurations, each GPU processes a specific segment of the output.
+            # The 'result' tensor represents the full output, which can vary in size based on
+            # the layer type (e.g., attention vs. feed-forward layers). We define the current
+            # segment using start_idx and end_idx. If the segment size doesn't match this GPU's
+            # slice of 'result', we create a zero tensor of the correct size for LoRA computation.
+            # This approach ensures accurate LoRA application across various layer sizes and
+            # configurations, adapting to different model architectures and parallelization strategies.
+            #
+            # Example scenarios where this is necessary:
+            # 1. The adapter's size doesn't evenly divide across GPUs.
+            # 2. We're processing the last segment which might be smaller.
+            # 3. Different projection layers (q, k, v) have different sizes.
+            if end_idx - start_idx != result.shape[1]:
+                proj = torch.zeros_like(result[:, start_idx:end_idx])
+            else:
+                proj = result
+
+            for r, rank_segments in data.rank_data.items():
+                lora_a_ptr = rank_segments.lora_a_ptr
+                lora_b_ptr = rank_segments.lora_b_ptr
+
+                if lora_a_ptr is None or lora_b_ptr is None:
+                    raise ValueError("LoRA data is missing")
+
+                if data.use_sgmv:
+                    # Use SGMV for prefill
+                    v = lora_a_sgmv_cutlass(
+                        input,
+                        rank_segments.tmp_shrink,
+                        lora_a_ptr,
+                        rank_segments.segment_starts,
+                        rank_segments.segment_ends,
+                        self.layer_id,
+                        r,
+                    )
+
+                    if self.process_group.size() > 1:
+                        v = self.collect_lora_a(v)
+
+                    lora_b_sgmv_cutlass(
+                        proj,
+                        v,
+                        rank_segments.tmp_expand,
+                        lora_b_ptr,
+                        rank_segments.segment_starts,
+                        rank_segments.segment_ends,
+                        self.layer_id,
+                    )
+                else:
+                    # Use BGMV for decode
+                    v = torch.zeros(
+                        (input.size(0), r), dtype=input.dtype, device=input.device
+                    )
+                    # TODO: error with [-1, 0], but not [0, -1]
+                    add_lora_a_bgmv(
+                        v,
+                        input,
+                        lora_a_ptr,
+                        rank_segments.indices,
+                        self.layer_id,
+                    )
+
+                    if self.process_group.size() > 1:
+                        v = self.collect_lora_a(v)
+
+                    add_lora_b_bgmv(
+                        proj,
+                        v,
+                        lora_b_ptr,
+                        rank_segments.indices,
+                        self.layer_id,
+                    )
+
+            if end_idx - start_idx != result.shape[1]:
+                result[:, start_idx:end_idx] += proj
+        else:
+            for adapter_index in adapter_data.meta.adapter_set:
+                if data is not None and data.has_adapter(adapter_index):
+                    adapter_mask = (
+                        (adapter_data.meta.adapter_indices == adapter_index)
+                        .to(input.dtype)
+                        .view(-1, 1)
+                    )
+                    layer_result = self.forward_lora(
+                        input, data, adapter_index, adapter_mask
+                    )
+                    result[:, start_idx:end_idx] += layer_result
+
+        return result
+
+    def forward_lora(
+        self,
+        input: torch.Tensor,
+        data: "BatchLoraWeights",
+        adapter_index: int,
+        adapter_mask: torch.Tensor,
+    ) -> torch.Tensor:
+        lora_a = data.lora_a[adapter_index][self.layer_id, :, :]
+        lora_b = data.lora_b[adapter_index][self.layer_id, :, :]
+
+        lora_a = orient_for_rank(lora_a, lora_b.size(0))
+
+        a_out = input @ lora_a
+        if self.process_group.size() > 1:
+            a_out = self.collect_lora_a(a_out)
+
+        result = (a_out @ lora_b) * adapter_mask
+        return result
+
+    def collect_lora_a(self, a_out: torch.Tensor) -> torch.Tensor:
+        raise NotImplementedError("Implemented in subclasses")
+
+
+class TensorParallelMultiAdapterLinear(LoraLinear):
+    def __init__(
+        self,
+        base_layer: nn.Module,
+        layer_id: int,
+        layer_names: List[str],
+        sizes: List[int],
+        process_group: ProcessGroup,
+    ):
+        super().__init__(base_layer, layer_id, process_group)
+        self.layer_names = layer_names
+        self.sizes = sizes
+
+    @classmethod
+    def load(
+        cls,
+        base_layer: nn.Module,
+        layer_id: int,
+        layer_names: List[str],
+        sizes: List[int],
+        process_group: ProcessGroup,
+    ):
+        return TensorParallelMultiAdapterLinear(
+            base_layer, layer_id, layer_names, sizes, process_group
+        )
+
+    def forward(
+        self, input: torch.Tensor, adapter_data: "AdapterBatchData"
+    ) -> torch.Tensor:
+        result = self.base_layer(input)
+
+        # noop if no layer names are provided (e.g. for models without adapters)
+        if self.layer_names is None:
+            return result
+
+        # handle models like Bloom that have inputs of shape
+        # (batch_size, sequence_length, hidden_size)
+        # we need to reshape them to (batch_size * sequence_length, hidden_size)
+        # for the LoRA computation, then reshape back
+        prev_shape = result.shape
+        is_3d = len(input.shape) >= 3
+        if is_3d:
+            input = input.reshape(-1, input.shape[-1])
+            result = result.reshape(-1, result.shape[-1])
+
+        offset = 0
+        for i, layer_name in enumerate(self.layer_names):
+            start_idx = offset // self.process_group.size()
+            # The 'sizes' parameter is essential in tensor-parallel setups for handling multiple
+            # projection layers (q_proj, k_proj, v_proj) by defining their output dimensions. It
+            # ensures correct slicing of the result tensor, accommodating variations like grouped-query
+            # attention where k_proj and v_proj differ from q_proj. This allows precise application of
+            # LoRA adapters to each sub-component of the multi-head attention mechanism, managing the
+            # different projection sizes across layers and model architectures.
+            if self.sizes is not None:
+                offset += self.sizes[i]
+                end_idx = offset // self.process_group.size()
+            else:
+                end_idx = result.shape[1]
+
+            result = self.forward_layer_type(
+                result, input, adapter_data, layer_name, start_idx, end_idx
+            )
+
+        if is_3d:
+            result = result.reshape(prev_shape)
+
+        return result
+
+    def collect_lora_a(self, a_out: torch.Tensor) -> torch.Tensor:
+        # Tensor parallel implementation of X @ A@B, where A and B are sharded column-wise.
+        # We use an all-gather between X@A and (X@A)@B to ensure alignment across ranks.
+        #
+        # TODO(travis): this is not very efficient as we do an all-gather for every adapter,
+        #   instead we could pre-allocate a (B, a, r) tensor for all adapters with the same
+        #   rank, compute `a_out` on each, and then slice them into the buffer as shown here:
+        #   https://discuss.pytorch.org/t/concatenate-tensors-without-memory-copying/34609
+        gathered_tensors = [
+            torch.empty_like(a_out) for _ in range(self.process_group.size())
+        ]
+        torch.distributed.all_gather(gathered_tensors, a_out)
+        return torch.cat(gathered_tensors, dim=1)
+
+
+class TensorParallelAdapterRowLinear(LoraLinear):
+    def __init__(self, base_layer, layer_id, layer_name, process_group):
+        super().__init__(base_layer, layer_id, process_group)
+        self.layer_name = layer_name
+
+    @classmethod
+    def load(cls, base_layer, layer_id, layer_name, process_group):
+        return cls(base_layer, layer_id, layer_name, process_group)
+
+    def forward(
+        self, input: torch.Tensor, adapter_data: "AdapterBatchData"
+    ) -> torch.Tensor:
+        result = self.base_layer(input)
+
+        if self.layer_name is None:
+            return result
+
+        # Fused all-gather + all-reduce from S-LoRA paper: https://arxiv.org/abs/2311.03285
+        stride = result.shape[-1] // self.process_group.size()
+        start_idx = self.process_group.rank() * stride
+        end_idx = (self.process_group.rank() + 1) * stride
+
+        self.forward_layer_type(
+            result, input, adapter_data, self.layer_name, start_idx, end_idx
+        )
+
+        return result
+
+    def collect_lora_a(self, a_out: torch.Tensor) -> torch.Tensor:
+        # Tensor parallel implementation of X @ A@B, where A and B are sharded row-wise.
+        # We use an all-reduce between X@A and (X@A)@B to ensure alignment across ranks.
+        #
+        # TODO(travis): this is not very efficient as we do an all-reduce for every adapter,
+        #   instead we could pre-allocate a (B, a, r) tensor for all adapters with the same
+        #   rank, compute `a_out` on each, and then slice them into the buffer as shown here:
+        #   https://discuss.pytorch.org/t/concatenate-tensors-without-memory-copying/34609
+        torch.distributed.all_reduce(a_out, group=self.process_group)
+        return a_out
diff --git a/server/text_generation_server/layers/marlin/__init__.py b/server/text_generation_server/layers/marlin/__init__.py
new file mode 100644
index 00000000..3ff3ed58
--- /dev/null
+++ b/server/text_generation_server/layers/marlin/__init__.py
@@ -0,0 +1,15 @@
+from text_generation_server.layers.marlin.fp8 import GPTQMarlinFP8Linear
+from text_generation_server.layers.marlin.gptq import (
+    GPTQMarlinWeightsLoader,
+    can_use_gptq_marlin,
+    repack_gptq_for_marlin,
+)
+from text_generation_server.layers.marlin.marlin import MarlinWeightsLoader
+
+__all__ = [
+    "GPTQMarlinFP8Linear",
+    "GPTQMarlinWeightsLoader",
+    "MarlinWeightsLoader",
+    "can_use_gptq_marlin",
+    "repack_gptq_for_marlin",
+]
diff --git a/server/text_generation_server/layers/marlin/fp8.py b/server/text_generation_server/layers/marlin/fp8.py
new file mode 100644
index 00000000..fe55a58a
--- /dev/null
+++ b/server/text_generation_server/layers/marlin/fp8.py
@@ -0,0 +1,140 @@
+from typing import Optional
+
+import torch
+import torch.nn as nn
+from loguru import logger
+from text_generation_server.layers.fp8 import fp8_quantize
+from text_generation_server.layers.marlin.gptq import _check_valid_shape
+from text_generation_server.layers.marlin.util import (
+    _check_marlin_kernels,
+    permute_scales,
+)
+from text_generation_server.utils.log import log_once
+
+try:
+    import marlin_kernels
+except ImportError:
+    marlin_kernels = None
+
+
+MARLIN_TILE_SIZE = 16
+
+
+class GPTQMarlinFP8Linear(nn.Module):
+    """
+    FP8 GPTQ-Marlin linear layer.
+    """
+
+    def __init__(
+        self,
+        qweight: torch.Tensor,
+        scales: torch.Tensor,
+        bias: Optional[torch.Tensor],
+    ) -> None:
+        super().__init__()
+
+        _check_marlin_kernels()
+        assert marlin_kernels is not None
+
+        log_once(logger.info, "GPU does not support FP8, using Marlin FP8 kernel")
+
+        scales = scales.unsqueeze(0)
+        if scales.shape[1] == 1:
+            out_features, in_features = qweight.shape
+            scales = scales.repeat(1, out_features)
+        qweight, scales = repack_fp8_for_marlin(qweight, scales)
+
+        in_features = qweight.shape[0] * MARLIN_TILE_SIZE
+        out_features = scales.shape[1]
+        _check_valid_shape(in_features=in_features, out_features=out_features)
+
+        self.qweight = qweight
+        self.scales = scales
+        self.bias = bias if bias is not None else None
+
+        self.workspace = torch.zeros(
+            out_features // 64 * 16, dtype=torch.int, device=qweight.device
+        )
+
+    @classmethod
+    def from_unquant(cls, weight, bias, dtype):
+        qweight, scales = fp8_quantize(weight)
+        return cls(qweight=qweight, scales=scales.to(dtype), bias=bias)
+
+    @classmethod
+    def from_fp8(cls, weight, scale, _input_scale, bias, dtype):
+        return cls(qweight=weight, scales=scale.to(dtype), bias=bias)
+
+    def forward(self, A: torch.Tensor) -> torch.Tensor:
+        assert marlin_kernels is not None
+
+        A_flat = A.view(-1, A.shape[-1])
+        C = marlin_kernels.fp8_marlin_gemm(
+            A_flat,
+            self.qweight,
+            self.scales,
+            self.workspace,
+            8,
+            A_flat.shape[0],
+            self.scales.shape[1],
+            A_flat.shape[1],
+        )
+        C = C.reshape(A.shape[:-1] + (self.scales.shape[1],))
+
+        if self.bias is not None:
+            C += self.bias
+
+        return C
+
+
+def pack_fp8_as_int32(fp8_tensor: torch.Tensor) -> torch.Tensor:
+    """
+    Repack FP8 weights to gptq format (packed int32 elements).
+    """
+    assert fp8_tensor.dtype == torch.float8_e4m3fn
+
+    if fp8_tensor.shape[0] % 4 != 0:
+        raise ValueError(
+            f"Leading tensor dimension is not divisable by 4: {fp8_tensor.shape[0]}"
+        )
+
+    # Reshape to prepare for packing
+    reshaped = fp8_tensor.reshape(-1, 4, *fp8_tensor.shape[1:])
+
+    # Convert fp8 to uint8 (byte) representation
+    byte_tensor = reshaped.view(torch.uint8)
+
+    # Pack 4 uint8 values into one int32
+    packed = torch.zeros(
+        fp8_tensor.shape[0] // 4,
+        fp8_tensor.shape[1],
+        dtype=torch.int32,
+        device=fp8_tensor.device,
+    )
+
+    for i in range(4):
+        packed.bitwise_or_(byte_tensor[:, i].to(torch.int32) << i * 8)
+
+    return packed
+
+
+def repack_fp8_for_marlin(weight: torch.Tensor, scales: torch.Tensor):
+    """
+    Repack FP8 tensor for GPTQ-Marlin.
+    """
+
+    out_features, in_features = weight.shape
+
+    # Torch linear layers weights with shape [out_features, in_features],
+    # GPTQ-quantized weights use [in_feateres/pack_factor, in_features],
+    # so transpose before packing.
+    qweight = pack_fp8_as_int32(weight.t())
+
+    perm = torch.empty(0, dtype=torch.int, device=qweight.device)
+    repacked = marlin_kernels.gptq_marlin_repack(
+        qweight, perm, in_features, out_features, 8
+    )
+
+    scales = permute_scales(scales)
+
+    return repacked, scales
diff --git a/server/text_generation_server/layers/marlin/gptq.py b/server/text_generation_server/layers/marlin/gptq.py
new file mode 100644
index 00000000..0a785d94
--- /dev/null
+++ b/server/text_generation_server/layers/marlin/gptq.py
@@ -0,0 +1,464 @@
+from dataclasses import dataclass
+from typing import List, Optional, Union
+
+import numpy
+import torch
+import torch.nn as nn
+from loguru import logger
+from text_generation_server.layers.marlin.util import (
+    _check_marlin_kernels,
+    marlin_zero_points,
+    permute_scales,
+    unpack_cols,
+)
+from text_generation_server.utils.import_utils import SYSTEM
+from text_generation_server.utils.log import log_once
+from text_generation_server.utils.weights import Weight, Weights, WeightsLoader
+
+try:
+    import marlin_kernels
+except ImportError:
+    marlin_kernels = None
+
+try:
+    major, _minor = torch.cuda.get_device_capability()
+    has_sm_8_0 = major >= 8
+except Exception:
+    has_sm_8_0 = False
+
+
+GPTQ_MARLIN_BITS = [4, 8]
+GPTQ_MARLIN_GROUP_SIZES = [-1, 32, 64, 128]
+MARLIN_TILE_SIZE = 16
+
+
+def can_use_gptq_marlin(
+    *, bits: int, groupsize: int, quant_method: str, quantize: str, sym: bool
+) -> bool:
+    return (
+        SYSTEM == "cuda"
+        and marlin_kernels is not None
+        and has_sm_8_0
+        and quantize in {"awq", "gptq"}
+        and quant_method in {"awq", "gptq"}
+        and bits in GPTQ_MARLIN_BITS
+        and groupsize in GPTQ_MARLIN_GROUP_SIZES
+        # We only suppord asymmetric quantization for AWQ.
+        and (sym or quant_method == "awq")
+    )
+
+
+class GPTQMarlinWeightsLoader(WeightsLoader):
+    """
+    Loader for using GPTQ- and AWQ-quantized weights with Marlin kernels.
+    """
+
+    def __init__(
+        self,
+        *,
+        bits: int,
+        desc_act: bool,
+        groupsize: int,
+        quant_method: str,
+        quantize: str,
+        sym: bool,
+    ):
+        self.bits = bits
+        self.desc_act = desc_act
+        self.groupsize = groupsize
+        self.quant_method = quant_method
+        self.quantize = quantize
+        self.sym = sym
+
+    def get_weights(self, weights: Weights, prefix: str):
+        log_once(logger.info, "Using GPTQ-Marlin kernels")
+        try:
+            qweight = weights.get_tensor(f"{prefix}.qweight")
+        except RuntimeError:
+            raise RuntimeError(
+                f"Cannot load `{self.quantize}` weight for GPTQ -> Marlin repacking, make sure the model is already quantized"
+            )
+
+        if not self.sym:
+            qzeros = weights.get_tensor(f"{prefix}.qzeros")
+        else:
+            qzeros = None
+
+        if self.quant_method == "awq":
+            g_idx = None
+        else:
+            g_idx = weights.get_tensor(f"{prefix}.g_idx")
+        scales = weights.get_tensor(f"{prefix}.scales")
+
+        return repack_gptq_for_marlin(
+            qweight=qweight,
+            scales=scales,
+            qzeros=qzeros,
+            g_idx=g_idx,
+            bits=self.bits,
+            desc_act=self.desc_act,
+            groupsize=self.groupsize,
+            quant_method=self.quant_method,
+            sym=self.sym,
+            sharded_infeatures=False,
+        )
+
+    def get_weights_col_packed(
+        self,
+        weights: Weights,
+        prefix: str,
+        block_sizes: Union[int, List[int]],
+    ):
+        try:
+            qweight = weights.get_packed_sharded(
+                f"{prefix}.qweight", dim=1, block_sizes=block_sizes
+            )
+        except RuntimeError:
+            raise RuntimeError(
+                f"Cannot load `{self.quantize}` weight, make sure the model is already quantized."
+            )
+        scales = weights.get_packed_sharded(
+            f"{prefix}.scales", dim=1, block_sizes=block_sizes
+        )
+        scales = scales.to(dtype=weights.dtype)
+
+        if not self.sym:
+            qzeros = weights.get_packed_sharded(
+                f"{prefix}.qzeros", dim=1, block_sizes=block_sizes
+            )
+        else:
+            qzeros = None
+
+        if self.quant_method == "awq":
+            g_idx = None
+        else:
+            g_idx = weights.get_tensor(f"{prefix}.g_idx")
+        return repack_gptq_for_marlin(
+            qweight=qweight,
+            scales=scales,
+            qzeros=qzeros,
+            g_idx=g_idx,
+            bits=self.bits,
+            desc_act=self.desc_act,
+            groupsize=self.groupsize,
+            quant_method=self.quant_method,
+            sym=self.sym,
+            sharded_infeatures=False,
+        )
+
+    def get_multi_weights_col(self, weights: Weights, prefixes: List[str], dim: int):
+        try:
+            qweight = torch.cat(
+                [weights.get_sharded(f"{p}.qweight", dim=1) for p in prefixes], dim=1
+            )
+        except RuntimeError:
+            raise RuntimeError(
+                f"Cannot load `{self.quantize}` weight, make sure the model is already quantized"
+            )
+
+        scales = torch.cat(
+            [weights.get_sharded(f"{p}.scales", dim=1) for p in prefixes], dim=1
+        )
+
+        if not self.sym:
+            qzeros = torch.cat(
+                [weights.get_sharded(f"{p}.qzeros", dim=1) for p in prefixes], dim=1
+            )
+        else:
+            qzeros = None
+
+        if self.quant_method == "awq":
+            g_idx = None
+        else:
+            w = [weights.get_tensor(f"{p}.g_idx") for p in prefixes]
+            for w2 in w[1:]:
+                torch.testing.assert_close(w2, w[0])
+            g_idx = w[0]
+
+        return repack_gptq_for_marlin(
+            qweight=qweight,
+            scales=scales,
+            qzeros=qzeros,
+            g_idx=g_idx,
+            bits=self.bits,
+            desc_act=self.desc_act,
+            groupsize=self.groupsize,
+            quant_method=self.quant_method,
+            sym=self.sym,
+            sharded_infeatures=False,
+        )
+
+    def get_weights_row(self, weights: Weights, prefix: str):
+        log_once(logger.info, "Using GPTQ-Marlin kernels")
+        try:
+            qweight = weights.get_sharded(f"{prefix}.qweight", dim=0)
+        except RuntimeError:
+            raise RuntimeError(
+                f"Cannot load `{self.quantize}` weight for GPTQ -> Marlin repacking, make sure the model is already quantized"
+            )
+
+        if not self.sym:
+            if self.desc_act or self.groupsize == -1:
+                qzeros = weights.get_tensor(f"{prefix}.qzeros")
+            else:
+                qzeros = weights.get_sharded(f"{prefix}.qzeros", dim=0)
+        else:
+            qzeros = None
+
+        if self.quant_method == "awq":
+            g_idx = None
+        else:
+            g_idx = weights.get_sharded(f"{prefix}.g_idx", dim=0)
+
+        if self.desc_act or self.groupsize == -1:
+            scales = weights.get_tensor(f"{prefix}.scales")
+        else:
+            scales = weights.get_sharded(f"{prefix}.scales", dim=0)
+
+        sharded_in_features = weights.process_group.size() > 1
+
+        return repack_gptq_for_marlin(
+            qweight=qweight,
+            scales=scales,
+            qzeros=qzeros,
+            g_idx=g_idx,
+            bits=self.bits,
+            desc_act=self.desc_act,
+            groupsize=self.groupsize,
+            quant_method=self.quant_method,
+            sym=self.sym,
+            sharded_infeatures=sharded_in_features,
+        )
+
+    def _get_gptq_params(self, weights: Weights):
+        if weights._has_tensor("gptq_bits") and weights._has_tensor("gptq_groupsize"):
+            self.bits = weights.get_tensor("gptq_bits").item()
+            self.groupsize = weights.get_tensor("gptq_groupsize").item()
+            self.desc_act = False
+            # `server quantize` used asymmetric quantization unconditionally
+            # before the `gptq_sym` setting tensor was added.
+            self.sym = (
+                weights.get_tensor("gptq_sym").item()
+                if weights._has_tensor("gptq_sym")
+                else False
+            )
+            self.quant_method = "gptq"
+
+
+@dataclass
+class GPTQMarlinWeight(Weight):
+    """
+    Repacked GPTQ Marlin weights.
+    """
+
+    qweight: torch.Tensor
+    qzeros: torch.Tensor
+    scales: torch.Tensor
+    g_idx: torch.Tensor
+    perm: torch.Tensor
+    bits: int
+    is_full_k: bool
+
+    def __post_init__(self):
+        assert self.qweight.dtype == torch.int32
+        assert self.scales.dtype == torch.float16
+        assert self.g_idx.dtype == torch.int32
+        assert self.perm.dtype == torch.int32
+
+    def get_linear(self, bias: torch.Tensor):
+        return GPTQMarlinLinear(
+            weight=self,
+            bias=bias,
+        )
+
+
+def repack_gptq_for_marlin(
+    *,
+    qweight: torch.Tensor,
+    qzeros: Optional[torch.Tensor],
+    scales: torch.Tensor,
+    g_idx: Optional[torch.Tensor],
+    bits: int,
+    desc_act: bool,
+    groupsize: int,
+    quant_method: str,
+    sym: bool,
+    sharded_infeatures: bool,
+) -> GPTQMarlinWeight:
+    """Convert GPTQ weights to a layout that's compatible with GPTQ-Marlin kernels."""
+    _check_marlin_kernels()
+    assert marlin_kernels is not None
+
+    if bits not in GPTQ_MARLIN_BITS:
+        supported_bits = ", ".join(str(b) for b in GPTQ_MARLIN_BITS)
+        raise RuntimeError(
+            f"Repacking {bits}-bit GPTQ weights as Marlin is not supported, must be one of: {supported_bits}"
+        )
+
+    if groupsize not in GPTQ_MARLIN_GROUP_SIZES:
+        supported_sizes = ", ".join(str(b) for b in GPTQ_MARLIN_GROUP_SIZES)
+        raise RuntimeError(
+            f"Repacking GPTQ weights with group size {groupsize} as Marlin is not supported, must be one of: {supported_sizes}"
+        )
+    if not (sym or quant_method == "awq"):
+        raise RuntimeError(
+            "Repacking GPTQ weights with asymmetric quantization as Marlin is not supported."
+        )
+
+    log_once(logger.info, f"Converting {quant_method} model to Marlin packing format.")
+
+    weights_per_int = 32 // bits
+    in_features = qweight.shape[0]
+    out_features = qweight.shape[1]
+
+    # AWQ uses column packing, GPTQ uses row packing
+    if quant_method == "awq":
+        out_features *= weights_per_int
+    else:
+        in_features *= weights_per_int
+
+    if in_features % groupsize != 0:
+        raise ValueError(
+            f"Number of input features ({in_features}) not divisible by group size ({groupsize})"
+        )
+
+    if g_idx is not None and desc_act and groupsize != -1:
+        perm = torch.argsort(g_idx).to(torch.int)
+        g_idx = g_idx[perm]
+    else:
+        perm = torch.empty(0, dtype=torch.int, device=qweight.device)
+        g_idx = torch.empty(0, dtype=torch.int, device=qweight.device)
+
+    if quant_method == "awq":
+        repacked = marlin_kernels.awq_marlin_repack(
+            qweight, in_features, out_features, bits
+        )
+        if qzeros is not None:
+            qzeros = awq_to_marlin_zero_points(
+                qzeros,
+                in_features // groupsize,
+                out_features,
+                bits,
+            )
+
+    else:
+        repacked = marlin_kernels.gptq_marlin_repack(
+            qweight, perm, in_features, out_features, bits
+        )
+
+    if qzeros is None:
+        qzeros = torch.empty(0, dtype=torch.int, device=qweight.device)
+
+    scales = permute_scales(scales)
+
+    is_full_k = not (desc_act and groupsize != -1 and sharded_infeatures)
+
+    return GPTQMarlinWeight(
+        qweight=repacked,
+        qzeros=qzeros,
+        scales=scales,
+        g_idx=g_idx,
+        perm=perm,
+        bits=bits,
+        is_full_k=is_full_k,
+    )
+
+
+class GPTQMarlinLinear(nn.Module):
+    """
+    Linear layer for GPTQ weights that were converted for the GPTQ-Marlin
+    kernels.
+    """
+
+    def __init__(
+        self,
+        *,
+        weight: GPTQMarlinWeight,
+        bias: Optional[torch.Tensor],
+    ):
+        super().__init__()
+
+        _check_marlin_kernels()
+        assert marlin_kernels is not None
+
+        in_features = weight.qweight.shape[0] * MARLIN_TILE_SIZE
+        out_features = weight.scales.shape[1]
+        _check_valid_shape(in_features=in_features, out_features=out_features)
+
+        self.bits = weight.bits
+        self.is_full_k = weight.is_full_k
+
+        self.qweight = weight.qweight
+        self.qzeros = weight.qzeros
+        self.scales = weight.scales
+        self.g_idx = weight.g_idx
+        self.perm = weight.perm
+        if bias is not None:
+            self.bias = bias
+        else:
+            self.bias = None
+
+        self.workspace = torch.zeros(
+            out_features // 64 * 16, dtype=torch.int, device=weight.qweight.device
+        )
+
+    def forward(self, A: torch.Tensor) -> torch.Tensor:
+        assert marlin_kernels is not None
+
+        A_flat = A.view(-1, A.shape[-1])
+        C = marlin_kernels.gptq_marlin_gemm(
+            A_flat,
+            self.qweight,
+            self.scales,
+            self.qzeros,
+            self.g_idx,
+            self.perm,
+            self.workspace,
+            self.bits,
+            A_flat.shape[0],
+            self.scales.shape[1],
+            A_flat.shape[1],
+            self.is_full_k,
+            self.qzeros.numel() > 0,
+            True,
+        )
+        C = C.reshape(A.shape[:-1] + (self.scales.shape[1],))
+
+        if self.bias is not None:
+            C += self.bias
+
+        return C
+
+
+def awq_to_marlin_zero_points(
+    q_zp_packed: torch.Tensor, size_k: int, size_n: int, num_bits: int
+) -> torch.Tensor:
+    # AWQ zero-points are quantized and packed on the column dim.
+    # In addition, the values are permuted based on dequantizer.
+    # Here we undo both of these, and then apply marlin permutation
+    # and pack it back.
+    q_zp = unpack_cols(q_zp_packed, num_bits, size_k, size_n)
+
+    # Undo interleaving (use argsort(..) to get inverse perm)
+    if num_bits == 4:
+        undo_interleave = numpy.argsort(numpy.array([0, 2, 4, 6, 1, 3, 5, 7]))
+    elif num_bits == 8:
+        undo_interleave = numpy.argsort(numpy.array([0, 2, 1, 3]))
+    else:
+        raise Exception("num_bits must be 4 or 8, got {}".format(num_bits))
+
+    q_zp = q_zp.reshape((-1, len(undo_interleave)))[:, undo_interleave].ravel()
+    q_zp = q_zp.reshape((-1, size_n)).contiguous()
+
+    marlin_zp = marlin_zero_points(q_zp, size_k, size_n, num_bits)
+    return marlin_zp
+
+
+def _check_valid_shape(in_features: int, out_features: int):
+    if (in_features % 128 != 0 or out_features % 64 != 0) and (
+        in_features % 64 != 0 or out_features % 128 != 0
+    ):
+        raise ValueError(
+            f"The GPTQ Marlin kernel does not have a valid thread configuration for weight matrix with shape ({out_features}, {in_features})."
+            " The shape elements must be divisible by (128, 64) or (64, 128)."
+        )
diff --git a/server/text_generation_server/layers/marlin/marlin.py b/server/text_generation_server/layers/marlin/marlin.py
new file mode 100644
index 00000000..89ebaca6
--- /dev/null
+++ b/server/text_generation_server/layers/marlin/marlin.py
@@ -0,0 +1,346 @@
+from dataclasses import dataclass
+from typing import List, Optional, Union
+
+import torch
+import torch.nn as nn
+from text_generation_server.layers.marlin.util import _check_marlin_kernels
+from text_generation_server.utils.weights import Weight, Weights, WeightsLoader
+
+try:
+    import marlin_kernels
+except ImportError:
+    marlin_kernels = None
+
+
+class MarlinWeightsLoader(WeightsLoader):
+    """Loader for Marlin-quantized weights."""
+
+    def __init__(self, *, bits: int, is_marlin_24: bool):
+        self.bits = bits
+        self.is_marlin_24 = is_marlin_24
+
+    def get_weights(self, weights: "Weights", prefix: str):
+        """
+        Get weights at the given prefix and apply without tensor paralllism.
+        """
+        is_marlin_24 = getattr(self, "gptq_checkpoint_format", None) == "marlin_24"
+        if is_marlin_24:
+            try:
+                B = weights.get_tensor(f"{prefix}.B_24")
+            except RuntimeError:
+                raise RuntimeError(
+                    "Cannot load `marlin` 2:4 sparsity weight, make sure the model is already quantized."
+                )
+
+            B_meta = weights.get_tensor(f"{prefix}.B_meta")
+            s = weights.get_tensor(f"{prefix}.s")
+            weight = GPTQMarlin24Weight(B=B, B_meta=B_meta, s=s, bits=self.bits)
+        else:
+            try:
+                B = weights.get_tensor(f"{prefix}.B")
+            except RuntimeError:
+                raise RuntimeError(
+                    "Cannot load `marlin` weight, make sure the model is already quantized."
+                )
+
+            s = weights.get_tensor(f"{prefix}.s")
+            weight = MarlinWeight(B=B, s=s)
+
+        return weight
+
+    def get_weights_col_packed(
+        self,
+        weights: Weights,
+        prefix: str,
+        block_sizes: Union[int, List[int]],
+    ):
+        if self.is_marlin_24:
+            B = weights.get_packed_sharded(
+                f"{prefix}.B_24", dim=1, block_sizes=block_sizes
+            )
+            B_meta = weights.get_packed_sharded(
+                f"{prefix}.B_meta", dim=1, block_sizes=block_sizes
+            )
+            s = weights.get_packed_sharded(
+                f"{prefix}.s", dim=1, block_sizes=block_sizes
+            )
+
+            weight = GPTQMarlin24Weight(B=B, B_meta=B_meta, s=s, bits=self.bits)
+        else:
+            B = weights.get_packed_sharded(
+                f"{prefix}.B", dim=1, block_sizes=block_sizes
+            )
+            s = weights.get_packed_sharded(
+                f"{prefix}.s", dim=1, block_sizes=block_sizes
+            )
+            weight = MarlinWeight(B=B, s=s)
+
+        return weight
+
+    def get_multi_weights_col(self, weights: Weights, prefixes: List[str], dim: int):
+        if self.is_marlin_24:
+            try:
+                B = torch.cat(
+                    [weights.get_sharded(f"{p}.B_24", dim=1) for p in prefixes], dim=1
+                )
+            except RuntimeError:
+                raise RuntimeError(
+                    "Cannot load `marlin` weight, make sure the model is already quantized"
+                )
+
+            B_meta = torch.cat(
+                [weights.get_sharded(f"{p}.B_meta", dim=1) for p in prefixes], dim=1
+            )
+
+            s = torch.cat(
+                [weights.get_sharded(f"{p}.s", dim=1) for p in prefixes], dim=1
+            )
+
+            weight = GPTQMarlin24Weight(B=B, B_meta=B_meta, s=s, bits=self.bits)
+        else:
+            try:
+                B = torch.cat(
+                    [weights.get_sharded(f"{p}.B", dim=1) for p in prefixes], dim=1
+                )
+            except RuntimeError:
+                raise RuntimeError(
+                    "Cannot load `marlin` weight, make sure the model is already quantized"
+                )
+            s = torch.cat(
+                [weights.get_sharded(f"{p}.s", dim=1) for p in prefixes], dim=1
+            )
+
+            weight = MarlinWeight(B=B, s=s)
+
+        return weight
+
+    def get_weights_row(self, weights: Weights, prefix: str):
+        if self.is_marlin_24:
+            try:
+                B = weights.get_sharded(f"{prefix}.B_24", dim=0)
+            except RuntimeError:
+                raise RuntimeError(
+                    "Cannot load `marlin` 2:4 sparsity weight, make sure the model is already quantized."
+                )
+
+            B_meta = weights.get_sharded(f"{prefix}.B_meta", dim=0)
+            num_groups = weights._get_slice(f"{prefix}.s").get_shape()[0]
+            if num_groups == 1:
+                # The number of groups is 1 when groupsize == -1. share
+                # scales between all shards in this case.
+                s = weights.get_tensor(f"{prefix}.s")
+            else:
+                s = weights.get_sharded(f"{prefix}.s", dim=0)
+
+            weight = GPTQMarlin24Weight(B=B, B_meta=B_meta, s=s, bits=self.bits)
+        else:
+            try:
+                B = weights.get_sharded(f"{prefix}.B", dim=0)
+            except RuntimeError:
+                raise RuntimeError(
+                    "Cannot load `marlin` weight, make sure the model is already quantized."
+                )
+
+            num_groups = weights._get_slice(f"{prefix}.s").get_shape()[0]
+            if num_groups == 1:
+                # The number of groups is 1 when groupsize == -1. share
+                # scales between all shards in this case.
+                s = weights.get_tensor(f"{prefix}.s")
+            else:
+                s = weights.get_sharded(f"{prefix}.s", dim=0)
+            weight = MarlinWeight(B=B, s=s)
+
+        return weight
+
+
+@dataclass
+class MarlinWeight(Weight):
+    """
+    Marlin weights.
+
+    Attributes:
+        B (torch.Tensor): int4-quantized weights packed into int32.
+        s (torch.Tensor): bfloat16/float16 scales.
+    """
+
+    B: torch.Tensor
+    s: torch.Tensor
+
+    def __post_init__(self):
+        assert self.B.dtype == torch.int32
+        assert self.s.dtype in [torch.float16, torch.bfloat16]
+
+    def get_linear(self, bias: torch.Tensor):
+        return MarlinLinear(weight=self, bias=bias)
+
+
+class MarlinLinear(nn.Module):
+    def __init__(self, *, weight: MarlinWeight, bias: Optional[torch.Tensor]):
+        super().__init__()
+
+        _check_marlin_kernels()
+        assert marlin_kernels is not None
+
+        in_features = weight.B.shape[0] * MARLIN_TILE_SIZE
+        out_features = weight.s.shape[1]
+        assert (
+            in_features % 128 == 0
+        ), f"Number of input features ({in_features}) not divisable by 128"
+        assert (
+            out_features % 256 == 0
+        ), f"Number of output features ({out_features}) not divisable by 256"
+
+        groupsize = -1 if weight.s.shape[0] == 1 else in_features // weight.s.shape[0]
+        assert groupsize in {
+            -1,
+            128,
+        }, f"Group size must be -1 or 128, was {groupsize}"
+
+        self.B = weight.B
+        self.s = weight.s
+        if bias is not None:
+            self.bias = bias
+        else:
+            self.bias = None
+
+        self.workspace = torch.zeros(
+            out_features // 64 * 16, dtype=torch.int, device=weight.B.device
+        )
+
+    def forward(self, A: torch.Tensor) -> torch.Tensor:
+        assert marlin_kernels is not None
+
+        C = marlin_kernels.marlin_gemm(
+            A.view(-1, A.shape[-1]),
+            self.B,
+            self.s,
+            self.workspace,
+            A.shape[0],
+            self.s.shape[1],
+            A.shape[1],
+        )
+        C = C.reshape(A.shape[:-1] + (self.s.shape[1],))
+
+        if self.bias is not None:
+            C += self.bias
+
+        return C
+
+
+GPTQ_MARLIN_24_MIN_THREAD_N = 128
+GPTQ_MARLIN_24_MIN_THREAD_K = 128
+GPTQ_MARLIN_24_MAX_PARALLEL = 64
+GPTQ_MARLIN_24_SUPPORTED_NUM_BITS = [4, 8]
+GPTQ_MARLIN_24_SUPPORTED_GROUP_SIZES = [-1, 128]
+MARLIN_TILE_SIZE = 16
+
+
+@dataclass
+class GPTQMarlin24Weight:
+    """
+    GPTQ-Marlin 2:4 weights.
+
+    Attributes:
+        B (torch.Tensor): int4-quantized weights packed into int32.
+        B_meta (torch.Tensor): metadata for 2:4 sparsity.
+        s (torch.Tensor): float16 scales.
+        bits: quantized weight size.
+    """
+
+    B: torch.Tensor
+    B_meta: torch.Tensor
+    s: torch.Tensor
+    bits: int
+
+    def __post_init__(self):
+        assert self.B.dtype == torch.int32
+        assert self.B_meta.dtype == torch.int16
+        assert self.s.dtype == torch.float16
+
+    def get_linear(self, bias: torch.Tensor):
+        return GPTQMarlin24Linear(
+            weight=self,
+            bias=bias,
+        )
+
+
+class GPTQMarlin24Linear(nn.Module):
+    def __init__(self, *, weight: GPTQMarlin24Weight, bias: Optional[torch.Tensor]):
+        super().__init__()
+
+        _check_marlin_kernels()
+        assert marlin_kernels is not None
+
+        if weight.bits not in GPTQ_MARLIN_24_SUPPORTED_NUM_BITS:
+            supported_bits = ", ".join(
+                str(b) for b in GPTQ_MARLIN_24_SUPPORTED_NUM_BITS
+            )
+            raise RuntimeError(
+                f"{weight.bits}-bit GPTQ Sparse 2:4 Marlin is not supported, must be one of: {supported_bits}"
+            )
+
+        in_features = weight.B.shape[0] * MARLIN_TILE_SIZE * 2
+        out_features = weight.s.shape[1]
+        groupsize = -1 if weight.s.shape[0] == 1 else in_features // weight.s.shape[0]
+
+        if groupsize not in GPTQ_MARLIN_24_SUPPORTED_GROUP_SIZES:
+            supported_sizes = ", ".join(
+                str(b) for b in GPTQ_MARLIN_24_SUPPORTED_GROUP_SIZES
+            )
+            raise RuntimeError(
+                f"Group size {groupsize} is not supported, must be one of: {supported_sizes}"
+            )
+
+        self.bits = weight.bits
+        weights_per_int32 = 32 // self.bits
+
+        assert (
+            out_features % GPTQ_MARLIN_24_MIN_THREAD_N == 0
+        ), f"Number of output features ({out_features}) not divisable by {GPTQ_MARLIN_24_MIN_THREAD_N} threads"
+        assert (
+            out_features % weights_per_int32 == 0
+        ), f"Number of output features ({out_features}) not divisable by weights per int32 ({weights_per_int32})"
+
+        assert (
+            in_features % GPTQ_MARLIN_24_MIN_THREAD_K == 0
+        ), f"Number of output features ({out_features}) not divisable by {GPTQ_MARLIN_24_MIN_THREAD_K} threads"
+        if groupsize != -1 and in_features % groupsize != 0:
+            raise ValueError(
+                f"Number of input features ({in_features}) not divisable by group size ({groupsize})"
+            )
+
+        self.B = weight.B
+        self.B_meta = weight.B_meta
+        self.s = weight.s
+        if bias is not None:
+            self.bias = bias
+        else:
+            self.bias = None
+
+        self.workspace = torch.zeros(
+            (out_features // GPTQ_MARLIN_24_MIN_THREAD_N) * GPTQ_MARLIN_24_MAX_PARALLEL,
+            dtype=torch.int,
+            device=weight.B.device,
+        )
+
+    def forward(self, A: torch.Tensor) -> torch.Tensor:
+        assert marlin_kernels is not None
+
+        C = marlin_kernels.gptq_marlin_24_gemm(
+            A.view(-1, A.shape[-1]),
+            self.B,
+            self.B_meta,
+            self.s,
+            self.workspace,
+            self.bits,
+            A.shape[0],
+            self.s.shape[1],
+            A.shape[1],
+        )
+
+        C = C.reshape(A.shape[:-1] + (self.s.shape[1],))
+
+        if self.bias is not None:
+            C += self.bias
+
+        return C
diff --git a/server/text_generation_server/layers/marlin/util.py b/server/text_generation_server/layers/marlin/util.py
new file mode 100644
index 00000000..250d1714
--- /dev/null
+++ b/server/text_generation_server/layers/marlin/util.py
@@ -0,0 +1,141 @@
+import functools
+from typing import List, Tuple
+
+import numpy
+import torch
+from text_generation_server.utils.import_utils import SYSTEM
+
+try:
+    import marlin_kernels
+except ImportError:
+    marlin_kernels = None
+
+try:
+    major, _minor = torch.cuda.get_device_capability()
+    has_sm_8_0 = major >= 8
+except Exception:
+    has_sm_8_0 = False
+
+
+def _check_marlin_kernels():
+    if not (SYSTEM == "cuda" and has_sm_8_0):
+        raise NotImplementedError(
+            "Using quantized Marlin models requires a GPU with CUDA capability 8.0 or later."
+        )
+
+    if marlin_kernels is None:
+        raise NotImplementedError(
+            "marlin is not installed, install it with: pip install server/marlin"
+        )
+
+
+# https://github.com/IST-DASLab/marlin/blob/2f6d7c10e124b3c5fa29ff8d77d568bd7af3274c/marlin/__init__.py#L40C1-L68C54
+@functools.cache
+def get_perms() -> Tuple[List[int], List[int]]:
+    scale_perm = []
+    for i in range(8):
+        scale_perm.extend([i + 8 * j for j in range(8)])
+    scale_perm_single = []
+    for i in range(4):
+        scale_perm_single.extend([2 * i + j for j in [0, 1, 8, 9, 16, 17, 24, 25]])
+    return scale_perm, scale_perm_single
+
+
+def permute_scales(scales: torch.Tensor):
+    scale_perm, scale_perm_single = get_perms()
+    out_features = scales.shape[1]
+    if scales.shape[0] == 1:
+        scales = scales.reshape((-1, len(scale_perm_single)))[:, scale_perm_single]
+    else:
+        scales = scales.reshape((-1, len(scale_perm)))[:, scale_perm]
+    return scales.reshape((-1, out_features)).contiguous()
+
+
+# Functions below are from vLLM
+
+
+def get_pack_factor(bits: int) -> int:
+    if 32 % bits != 0:
+        raise ValueError(f"Cannot {bits} bit values into uint32")
+    return 32 // bits
+
+
+def pack_cols(
+    q_w: torch.Tensor,
+    num_bits: int,
+    size_k: int,
+    size_n: int,
+):
+    assert q_w.shape == (size_k, size_n)
+
+    pack_factor = get_pack_factor(num_bits)
+    assert size_n % pack_factor == 0
+
+    orig_device = q_w.device
+
+    q_w = q_w.cpu().numpy().astype(numpy.uint32)
+
+    q_res = numpy.zeros((size_k, size_n // pack_factor), dtype=numpy.uint32)
+
+    for i in range(pack_factor):
+        q_res |= q_w[:, i::pack_factor] << num_bits * i
+
+    q_res = torch.from_numpy(q_res.astype(numpy.int32)).to(orig_device)
+    q_res = q_res.contiguous()
+
+    return q_res
+
+
+def unpack_cols(
+    packed_q_w: torch.Tensor,
+    num_bits: int,
+    size_k: int,
+    size_n: int,
+):
+    pack_factor = get_pack_factor(num_bits)
+    assert size_n % pack_factor == 0
+    assert packed_q_w.shape == (
+        size_k,
+        size_n // pack_factor,
+    ), "packed_q_w.shape = {} size_k = {}, size_n = {} pack_Factor = {}".format(
+        packed_q_w.shape, size_k, size_n, pack_factor
+    )
+
+    orig_device = packed_q_w.device
+
+    packed_q_w_cpu = packed_q_w.cpu().numpy().astype(numpy.uint32)
+    q_res = numpy.zeros((size_k, size_n), dtype=numpy.uint32)
+
+    mask = (1 << num_bits) - 1
+    for i in range(pack_factor):
+        vals = packed_q_w_cpu & mask
+        packed_q_w_cpu >>= num_bits
+        q_res[:, i::pack_factor] = vals
+
+    q_res = torch.from_numpy(q_res.astype(numpy.int32)).to(orig_device)
+    q_res = q_res.contiguous()
+
+    return q_res
+
+
+def marlin_zero_points(
+    zp: torch.Tensor, size_k: int, size_n: int, num_bits: int
+) -> torch.Tensor:
+    scale_perm, _ = get_perms()
+    # Permute zero-points in a similar way to scales, but do not use the
+    # "single" permutation, since zero-points are applied on every MMA
+    zp = zp.reshape((-1, len(scale_perm)))[:, scale_perm]
+
+    # Interleave column dim (for the dequantize code) and pack it to int32
+    if num_bits == 4:
+        interleave = numpy.array([0, 2, 4, 6, 1, 3, 5, 7])
+    elif num_bits == 8:
+        interleave = numpy.array([0, 2, 1, 3])
+    else:
+        raise Exception("num_bits must be 4 or 8, got {}".format(num_bits))
+
+    zp = zp.reshape((-1, len(interleave)))[:, interleave].ravel()
+    zp = zp.reshape((-1, size_n)).contiguous()
+    zp = pack_cols(zp, num_bits, size_k, size_n)
+
+    return zp
diff --git a/server/text_generation_server/layers/medusa.py b/server/text_generation_server/layers/medusa.py
index 7579ccdb..139c4dc2 100644
--- a/server/text_generation_server/layers/medusa.py
+++ b/server/text_generation_server/layers/medusa.py
@@ -32,6 +32,8 @@ class MedusaModel(torch.nn.Module):
         )
 
     def forward(self, x):
+        if not self.heads:
+            return None
         speculative_logits = torch.stack([head(x) for head in self.heads], dim=1)
         return speculative_logits
 
diff --git a/server/text_generation_server/layers/mlp.py b/server/text_generation_server/layers/mlp.py
index f08cb673..d33b41f3 100644
--- a/server/text_generation_server/layers/mlp.py
+++ b/server/text_generation_server/layers/mlp.py
@@ -45,12 +45,107 @@ class MLPSpeculatorLayerNorm(nn.Module):
         return x
 
 
+INV_SQRT2 = 2**-0.5
+
+
+def simple_norm(x: torch.Tensor, eps=1e-06):
+    xf = x
+    xf = xf * torch.rsqrt(xf.pow(2).mean(-1, keepdim=True) + eps)
+    x = xf.type_as(x)
+    return x * INV_SQRT2
+
+
+class MLPSpeculatorModelTied(torch.nn.Module):
+    def __init__(self, config, prefix, weights):
+        super().__init__()
+        self.config = config
+        self.n_predict = get_speculate()
+        self.hidden_size = config.hidden_size
+
+        self.emb = TensorParallelEmbedding(f"{prefix}.emb.0", weights)
+        self.proj0 = FastLinear.load(
+            config,
+            prefix=f"{prefix}.proj.0",
+            weights=weights,
+            bias=False,
+        )
+        self.proj1 = FastLinear.load(
+            config,
+            prefix=f"{prefix}.proj.1",
+            weights=weights,
+            bias=False,
+        )
+        self.head = FastLinear.load(config, f"{prefix}.head.0", weights, bias=False)
+        self.ln = MLPSpeculatorLayerNorm(
+            prefix=f"{prefix}.ln.0",
+            config=config,
+            weights=weights,
+        )
+
+        # Weights ensure that state_0 accounts for 50% of state magnitude by final head in expectation
+        self.state_weight = 0.5 ** (0.5 / self.n_predict) if self.n_predict > 0 else 1
+        self.activation = nn.GELU()
+        self.vsize = config.vocab_size
+        self.inner_dim = config.speculator_config["inner_dim"]
+        self.top_k_tokens_per_head = [1] * self.n_predict
+        self.emb_weight = math.sqrt(1 - self.state_weight**2) * math.sqrt(
+            self.inner_dim / 2
+        )
+        self.emb.weight *= self.emb_weight
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        input_ids: torch.Tensor,
+    ):
+        top_k_tokens_per_head = self.top_k_tokens_per_head
+
+        # k indicates # of candidates
+        # h indicates # of generated tokens
+        state = hidden_states
+        b = state.size(0)
+        ind = input_ids.unsqueeze(0)
+        all_probs = torch.empty(
+            b, self.n_predict, self.vsize, device=state.device
+        )  # b k h v
+        assert (
+            len(top_k_tokens_per_head) == self.n_predict
+        ), f"You must provide a topk number for each head ({self.n_predict} heads, {len(top_k_tokens_per_head)} provided)"
+        for i in range(self.n_predict):
+            # Project and predict
+            z = self.emb(ind)
+            # z = z.mul(self.emb_weight)  # b k d
+            if i == 0:
+                state = self.proj0(state) * self.state_weight + z
+            else:
+                state = self.proj1(state) * self.state_weight + z
+            state = self.activation(self.ln(state))  # b k d
+            probs = F.log_softmax(self.head(state), dim=-1)  # b k v
+            _probs, preds = probs.topk(top_k_tokens_per_head[i], dim=-1)  # b k k'
+
+            # Update candidate set with new predictions
+
+            # Update distribution set with new logits
+            all_probs[:, i] = probs.exp()
+
+            # Update state, log_probs and ind for new predictions
+            state = state.unsqueeze(2).expand(
+                -1, -1, top_k_tokens_per_head[i], -1
+            )  # b k k' d
+            state = state.reshape(-1, b, state.size(3))  # b kk' d
+            ind = preds.view(-1, b)  # b kk'
+
+        speculative_logits = all_probs
+        return speculative_logits
+
+
 class MLPSpeculatorModel(torch.nn.Module):
     def __init__(self, config, prefix, weights):
         super().__init__()
         self.config = config
         self.n_predict = get_speculate()
         self.hidden_size = config.hidden_size
+
         self.emb = nn.ModuleList(
             [
                 TensorParallelEmbedding(f"{prefix}.emb.{i}", weights)
@@ -84,13 +179,15 @@ class MLPSpeculatorModel(torch.nn.Module):
         )
 
         # Weights ensure that state_0 accounts for 50% of state magnitude by final head in expectation
-        self.state_weight = 0.5 ** (0.5 / self.n_predict)
-        self.emb_weight = math.sqrt(1 - self.state_weight**2)
+        self.state_weight = 0.5 ** (0.5 / self.n_predict) if self.n_predict > 0 else 1
         self.activation = nn.GELU()
-        # TODO
         self.vsize = config.vocab_size
         self.inner_dim = config.speculator_config["inner_dim"]
         self.top_k_tokens_per_head = [1] * self.n_predict
+        self.emb_weight = math.sqrt(1 - self.state_weight**2) * math.sqrt(
+            self.inner_dim / 2
+        )
+        self.emb.weight *= self.emb_weight
 
     def forward(
         self,
@@ -113,7 +210,7 @@ class MLPSpeculatorModel(torch.nn.Module):
         for i in range(self.n_predict):
             # Project and predict
             z = self.emb[i](ind)
-            z = z.mul(self.emb_weight * math.sqrt(self.inner_dim / 2))  # b k d
+            # z = z.mul(self.emb_weight)  # b k d
             state = self.proj[i](state) * self.state_weight + z
             state = self.activation(self.ln[i](state))  # b k d
             probs = F.log_softmax(self.head[i](state), dim=-1)  # b k v
@@ -136,10 +233,11 @@ class MLPSpeculatorModel(torch.nn.Module):
 
 
 class MLPSpeculatorHead(nn.Module):
-    def __init__(self, lm_head, mlp_speculator):
+    def __init__(self, lm_head, mlp_speculator, scale_input: bool):
         super().__init__()
         self.lm_head = lm_head
         self.mlp_speculator = mlp_speculator
+        self.scale_input = scale_input
 
     def forward(
         self, input: torch.Tensor
@@ -150,6 +248,8 @@ class MLPSpeculatorHead(nn.Module):
             return logits, None
 
         input_ids = logits.argmax(dim=-1)
+        if self.scale_input:
+            input = simple_norm(input)
         speculative_logits = self.mlp_speculator(input, input_ids)
         return logits, speculative_logits
 
@@ -171,6 +271,12 @@ class MLPSpeculatorHead(nn.Module):
                         )
                     routing[k] = filename
 
-        mlp_speculator = MLPSpeculatorModel(config, "speculator", weights)
+        tie_weights = config.speculator_config.get("tie_weights", False)
+        if tie_weights:
+            mlp_speculator = MLPSpeculatorModelTied(config, "speculator", weights)
+        else:
+            mlp_speculator = MLPSpeculatorModel(config, "speculator", weights)
+        # This is used in https://huggingface.co/ibm-fms/llama3-70b-accelerator
+        scale_input = config.speculator_config.get("scale_input", False)
         lm_head = TensorParallelHead.load(config, prefix, weights)
-        return MLPSpeculatorHead(lm_head, mlp_speculator)
+        return MLPSpeculatorHead(lm_head, mlp_speculator, scale_input)
diff --git a/server/text_generation_server/layers/moe/__init__.py b/server/text_generation_server/layers/moe/__init__.py
new file mode 100644
index 00000000..2c46ca02
--- /dev/null
+++ b/server/text_generation_server/layers/moe/__init__.py
@@ -0,0 +1,257 @@
+from typing import Optional, Protocol, runtime_checkable
+
+import torch
+import torch.nn as nn
+from loguru import logger
+from transformers.activations import ACT2FN
+
+from text_generation_server.layers import (
+    TensorParallelColumnLinear,
+    TensorParallelRowLinear,
+)
+from text_generation_server.layers.fp8 import HybridFP8UnquantLoader
+from text_generation_server.layers.marlin import GPTQMarlinWeightsLoader
+from text_generation_server.layers.moe.gptq_marlin import (
+    GPTQMarlinSparseMoELayer,
+    can_use_marlin_moe_gemm,
+)
+from text_generation_server.layers.moe.unquantized import UnquantizedSparseMoELayer
+from text_generation_server.utils.import_utils import SYSTEM
+from text_generation_server.utils.log import log_once
+from text_generation_server.utils.weights import (
+    DefaultWeightsLoader,
+    Weights,
+    UnquantizedWeight,
+)
+
+if SYSTEM == "rocm":
+    from .fused_moe_rocm import grouped_topk
+    from vllm.model_executor.layers.fused_moe import fused_topk
+elif SYSTEM != "ipex":
+    from moe_kernels.fused_moe import fused_topk, grouped_topk
+
+
+# NOTE: we are using a protocol here, because multiple inherance is not nice.
+#       We need `Module`, and `Module` -> some abstract class -> some concrete
+#       class inheritance is whacky.
+
+
+@runtime_checkable
+class MoELayer(Protocol):
+    def __init__(
+        self,
+        *,
+        n_expert_group: Optional[int],
+        n_experts: int,
+        prefix: str,
+        renormalize: bool,
+        topk: int,
+        topk_group: Optional[int],
+        weights: Weights,
+        gate_proj_name: str = "gate_proj",
+        up_proj_name: str = "up_proj",
+        down_proj_name: str = "down_proj",
+        hidden_act: str = "silu",
+    ): ...
+
+    def forward(
+        self, x: torch.Tensor, *, gating_output: torch.Tensor
+    ) -> torch.Tensor: ...
+
+
+class DenseMoELayer(nn.Module):
+    """
+    Layer for MoE that applies *all* experts to each tokens and then weights
+    their outputs based on the calculated routing. This layer is much slower
+    than `SparseMoELayer` and should only be used when no fused kernels are
+    available (e.g. for unsupported quantizers).
+    """
+
+    def __init__(
+        self,
+        *,
+        n_expert_group: Optional[int],
+        n_experts: int,
+        prefix: str,
+        renormalize: bool,
+        topk: int,
+        topk_group: Optional[int],
+        weights: Weights,
+        gate_proj_name: str = "gate_proj",
+        up_proj_name: str = "up_proj",
+        down_proj_name: str = "down_proj",
+        hidden_act: str = "silu",
+    ):
+        super().__init__()
+
+        log_once(
+            logger.info,
+            "No fused layers are available for this model type, using (slower) dense MoE layer",
+        )
+
+        assert (n_expert_group is None) == (
+            topk_group is None
+        ), "n_expert_group and topk_group must both be None or have some value"
+
+        self.n_expert_group = n_expert_group
+        self.n_experts = n_experts
+        self.renormalize = renormalize
+        self.topk = topk
+        self.topk_group = topk_group
+
+        if "gelu" in hidden_act:
+            self.act = lambda x: torch.nn.functional.gelu(
+                x,
+                approximate=(
+                    "tanh"
+                    if hidden_act in ["gelu_fast", "gelu_pytorch_tanh"]
+                    else "none"
+                ),
+            )
+        elif "silu" in hidden_act:
+            self.act = torch.nn.functional.silu
+        else:
+            self.act = ACT2FN[hidden_act]
+
+        self.gate_proj = [
+            TensorParallelColumnLinear.load(
+                None,
+                prefix=f"{prefix}.{i}.{gate_proj_name}",
+                weights=weights,
+                bias=False,
+            )
+            for i in range(self.n_experts)
+        ]
+        self.up_proj = [
+            TensorParallelColumnLinear.load(
+                None,
+                prefix=f"{prefix}.{i}.{up_proj_name}",
+                weights=weights,
+                bias=False,
+            )
+            for i in range(self.n_experts)
+        ]
+        self.down_proj = [
+            TensorParallelRowLinear.load(
+                None,
+                prefix=f"{prefix}.{i}.{down_proj_name}",
+                weights=weights,
+                bias=False,
+            )
+            for i in range(self.n_experts)
+        ]
+
+        self.process_group = weights.process_group
+
+    def forward(self, x: torch.Tensor, *, gating_output: torch.Tensor) -> torch.Tensor:
+        """
+        x: (sequence_length, model_dim)
+        gating_output: (sequence_length, n_experts)
+        """
+        # optional reshape
+        input_shape = x.shape
+        x = x.view(-1, input_shape[-1])
+
+        if self.n_expert_group is not None and self.topk_group is not None:
+            topk_weights, topk_ids = grouped_topk(
+                x,
+                gating_output,
+                self.topk,
+                renormalize=self.renormalize,
+                num_expert_group=self.n_expert_group,
+                topk_group=self.topk_group,
+            )
+        else:
+            topk_weights, topk_ids = fused_topk(
+                x, gating_output, self.topk, self.renormalize
+            )
+            topk_weights = topk_weights.to(x.dtype)
+
+        weights = torch.zeros(
+            topk_ids.shape[0], self.n_experts, dtype=x.dtype, device=x.device
+        )
+
+        weights.scatter_(1, topk_ids.long(), topk_weights.to(weights.dtype))
+
+        out = torch.zeros_like(x)
+        for i in range(self.n_experts):
+            h = self.act(self.gate_proj[i](x)) * self.up_proj[i](x)
+            h = self.down_proj[i](h, reduce=False)
+            out += h * weights[:, i].view(-1, 1)
+
+        return out
+
+
+class SparseMoELayer(nn.Module):
+    """
+    Layer for MoE that uses fused kernels to only apply the active experts
+    for each token (rather than applying all experts and selecting the
+    outputs of active experts).
+    """
+
+    def __init__(
+        self,
+        *,
+        n_expert_group: Optional[int],
+        n_experts: int,
+        prefix: str,
+        renormalize: bool,
+        topk: int,
+        topk_group: Optional[int],
+        weights: Weights,
+        gate_proj_name: str = "gate_proj",
+        up_proj_name: str = "up_proj",
+        down_proj_name: str = "down_proj",
+    ):
+        super().__init__()
+
+        if (
+            isinstance(weights.loader, DefaultWeightsLoader)
+            and isinstance(weights.loader.weight_class, UnquantizedWeight)
+        ) or isinstance(weights.loader, HybridFP8UnquantLoader):
+            cls = UnquantizedSparseMoELayer
+        elif isinstance(weights.loader, GPTQMarlinWeightsLoader) and weights.loader.sym:
+            cls = GPTQMarlinSparseMoELayer
+        else:
+            raise ValueError(
+                f"Unsupported weights loader: {weights.loader}, sparse MoE is only supported for unquantized and GPTQ weights"
+            )
+
+        log_once(
+            logger.info,
+            "Using MoE layer wih fused gemm",
+        )
+
+        self.moe = cls(
+            n_expert_group=n_expert_group,
+            n_experts=n_experts,
+            prefix=prefix,
+            renormalize=renormalize,
+            topk=topk,
+            topk_group=topk_group,
+            weights=weights,
+            gate_proj_name=gate_proj_name,
+            up_proj_name=up_proj_name,
+            down_proj_name=down_proj_name,
+        )
+
+    def forward(self, x: torch.Tensor, *, gating_output: torch.Tensor) -> torch.Tensor:
+        return self.moe(x, gating_output=gating_output)
+
+    @staticmethod
+    def is_supported(weights: Weights) -> bool:
+        return (
+            (
+                isinstance(weights.loader, DefaultWeightsLoader)
+                and isinstance(weights.loader.weight_class, UnquantizedWeight)
+            )
+            or isinstance(weights.loader, HybridFP8UnquantLoader)
+            or (
+                isinstance(weights.loader, GPTQMarlinWeightsLoader)
+                and can_use_marlin_moe_gemm(
+                    quant_method=weights.loader.quant_method,
+                    quantize=weights.loader.quantize,
+                    sym=weights.loader.sym,
+                )
+            )
+        )
diff --git a/server/text_generation_server/layers/moe/fused_moe_rocm.py b/server/text_generation_server/layers/moe/fused_moe_rocm.py
new file mode 100644
index 00000000..68accb99
--- /dev/null
+++ b/server/text_generation_server/layers/moe/fused_moe_rocm.py
@@ -0,0 +1,52 @@
+# coding=utf-8
+# Copyright 2023, 2024 DeepSeek-AI and The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Tuple
+
+import torch
+import torch.distributed
+
+
+# TODO: Remove the functions once moe_kernel are built for ROCM
+def grouped_topk(
+    hidden_states: torch.Tensor,
+    gating_output: torch.Tensor,
+    topk: int,
+    renormalize: bool,
+    num_expert_group: int = 0,
+    topk_group: int = 0,
+) -> Tuple[torch.Tensor, torch.Tensor]:
+    scores = torch.softmax(gating_output, dim=-1)
+    num_token = scores.shape[0]
+    group_scores = (
+        scores.view(num_token, num_expert_group, -1).max(dim=-1).values
+    )  # [n, n_group]
+    group_idx = torch.topk(group_scores, k=topk_group, dim=-1, sorted=False)[
+        1
+    ]  # [n, top_k_group]
+    group_mask = torch.zeros_like(group_scores)  # [n, n_group]
+    group_mask.scatter_(1, group_idx, 1)  # [n, n_group]
+    score_mask = (
+        group_mask.unsqueeze(-1)
+        .expand(num_token, num_expert_group, scores.shape[-1] // num_expert_group)
+        .reshape(num_token, -1)
+    )  # [n, e]
+    tmp_scores = scores.masked_fill(~score_mask.bool(), 0.0)  # [n, e]
+    topk_weights, topk_ids = torch.topk(tmp_scores, k=topk, dim=-1, sorted=False)
+
+    if renormalize:
+        topk_weights = topk_weights / topk_weights.sum(dim=-1, keepdim=True)
+
+    return topk_weights, topk_ids
diff --git a/server/text_generation_server/layers/moe/gptq_marlin.py b/server/text_generation_server/layers/moe/gptq_marlin.py
new file mode 100644
index 00000000..3217cdc2
--- /dev/null
+++ b/server/text_generation_server/layers/moe/gptq_marlin.py
@@ -0,0 +1,215 @@
+from dataclasses import dataclass
+from typing import List, Optional
+
+import torch
+import torch.nn as nn
+
+from text_generation_server.utils.import_utils import SYSTEM
+from text_generation_server.utils.weights import Weights
+from text_generation_server.layers.marlin.gptq import (
+    GPTQMarlinWeight,
+    GPTQMarlinWeightsLoader,
+)
+
+if SYSTEM == "cuda":
+    from moe_kernels.fused_marlin_moe import fused_marlin_moe
+else:
+    fused_marlin_moe = None
+
+
+try:
+    major, _minor = torch.cuda.get_device_capability()
+    has_sm_8_0 = major >= 8
+except Exception:
+    has_sm_8_0 = False
+
+
+def can_use_marlin_moe_gemm(
+    *,
+    quant_method: str,
+    quantize: str,
+    sym: bool,
+):
+    return (
+        SYSTEM == "cuda"
+        and fused_marlin_moe is not None
+        and has_sm_8_0
+        and quantize == "gptq"
+        and quant_method == "gptq"
+        and sym
+    )
+
+
+@dataclass
+class GPTQMarlinMoEWeight:
+    qweight: torch.Tensor
+    qzeros: torch.Tensor
+    scales: torch.Tensor
+    g_idx: torch.Tensor
+    perm: torch.Tensor
+    is_full_k: bool
+
+
+class GPTQMarlinSparseMoELayer(nn.Module):
+    """
+    MoE layer that uses a fused GPTQ-Marlin kernel.
+    """
+
+    def __init__(
+        self,
+        *,
+        n_expert_group: Optional[int],
+        n_experts: int,
+        prefix: str,
+        renormalize: bool,
+        topk: int,
+        topk_group: Optional[int],
+        weights: Weights,
+        gate_proj_name: str = "gate_proj",
+        up_proj_name: str = "up_proj",
+        down_proj_name: str = "down_proj",
+    ):
+        super().__init__()
+
+        if not (
+            isinstance(weights.loader, GPTQMarlinWeightsLoader) and weights.loader.sym
+        ):
+            raise ValueError(
+                f"Unsupported weights loader: {weights.loader}, only GPTQMarlinWeightsLoader with symmetric quantization is supported"
+            )
+
+        assert (n_expert_group is None) == (
+            topk_group is None
+        ), "n_expert_group and topk_group must both be None or have some value"
+
+        self.n_expert_group = n_expert_group
+        self.topk = topk
+        self.topk_group = topk_group
+        self.renormalize = renormalize
+
+        self.gate_up_proj = _load_expert_multi_weights_col(
+            prefix=prefix,
+            n_experts=n_experts,
+            names=[gate_proj_name, up_proj_name],
+            weights=weights,
+        )
+
+        self.down_proj = _load_expert_weights_row(
+            prefix=prefix, n_experts=n_experts, name=down_proj_name, weights=weights
+        )
+
+        self.bits = weights.loader.bits
+
+    def forward(self, x: torch.Tensor, *, gating_output: torch.Tensor) -> torch.Tensor:
+        return fused_marlin_moe(
+            x,
+            w1=self.gate_up_proj.qweight,
+            w2=self.down_proj.qweight,
+            g_idx1=self.gate_up_proj.g_idx,
+            g_idx2=self.down_proj.g_idx,
+            perm1=self.gate_up_proj.perm,
+            perm2=self.down_proj.perm,
+            w1_scale=self.gate_up_proj.scales,
+            w2_scale=self.down_proj.scales,
+            is_full_k1=self.gate_up_proj.is_full_k,
+            is_full_k2=self.down_proj.is_full_k,
+            gating_output=gating_output,
+            topk=self.topk,
+            renormalize=self.renormalize,
+            use_grouped_topk=self.n_expert_group is not None,
+            num_expert_group=self.n_expert_group,
+            topk_group=self.topk_group,
+            num_bits=self.bits,
+        )
+
+
+def _load_expert_multi_weights_col(
+    *,
+    prefix: str,
+    n_experts: int,
+    names: List[str],
+    weights: Weights,
+) -> GPTQMarlinMoEWeight:
+    moe_weight = None
+    for i in range(n_experts):
+        weight = weights.get_multi_weights_col(
+            [f"{prefix}.{i}.{name}" for name in names], 0
+        )
+        assert isinstance(weight, GPTQMarlinWeight)
+        moe_weight = _pack_weight(
+            n_experts=n_experts, expert=i, weight=weight, moe_weight=moe_weight
+        )
+    assert moe_weight is not None
+    return moe_weight
+
+
+def _load_expert_weights_row(
+    *,
+    prefix: str,
+    n_experts: int,
+    name: str,
+    weights: Weights,
+) -> GPTQMarlinMoEWeight:
+    moe_weight = None
+    for i in range(n_experts):
+        weight = weights.get_weights_row(
+            f"{prefix}.{i}.{name}",
+        )
+        assert isinstance(weight, GPTQMarlinWeight)
+        moe_weight = _pack_weight(
+            n_experts=n_experts, expert=i, weight=weight, moe_weight=moe_weight
+        )
+    assert moe_weight is not None
+    return moe_weight
+
+
+def _pack_weight(
+    *,
+    n_experts: int,
+    expert: int,
+    moe_weight: Optional[GPTQMarlinMoEWeight],
+    weight: GPTQMarlinWeight,
+) -> GPTQMarlinMoEWeight:
+    if moe_weight is None:
+        qweight = torch.empty(
+            (n_experts,) + weight.qweight.shape,
+            dtype=weight.qweight.dtype,
+            device=weight.qweight.device,
+        )
+        qzeros = torch.empty(
+            (n_experts,) + weight.qzeros.shape,
+            dtype=weight.qzeros.dtype,
+            device=weight.qzeros.device,
+        )
+        scales = torch.empty(
+            (n_experts,) + weight.scales.shape,
+            dtype=weight.scales.dtype,
+            device=weight.scales.device,
+        )
+        g_idx = torch.empty(
+            (n_experts,) + weight.g_idx.shape,
+            dtype=weight.g_idx.dtype,
+            device=weight.g_idx.device,
+        )
+        perm = torch.empty(
+            (n_experts,) + weight.perm.shape,
+            dtype=weight.perm.dtype,
+            device=weight.perm.device,
+        )
+
+        moe_weight = GPTQMarlinMoEWeight(
+            qweight=qweight,
+            qzeros=qzeros,
+            scales=scales,
+            g_idx=g_idx,
+            perm=perm,
+            is_full_k=weight.is_full_k,
+        )
+
+    moe_weight.qweight[expert] = weight.qweight
+    moe_weight.qzeros[expert] = weight.qzeros
+    moe_weight.scales[expert] = weight.scales
+    moe_weight.g_idx[expert] = weight.g_idx
+    moe_weight.perm[expert] = weight.perm
+
+    return moe_weight
diff --git a/server/text_generation_server/layers/moe/unquantized.py b/server/text_generation_server/layers/moe/unquantized.py
new file mode 100644
index 00000000..d9d62c0e
--- /dev/null
+++ b/server/text_generation_server/layers/moe/unquantized.py
@@ -0,0 +1,138 @@
+from typing import Optional
+
+import torch
+import torch.nn as nn
+
+from text_generation_server.utils.import_utils import SYSTEM
+from text_generation_server.utils.weights import UnquantizedWeight, Weights
+
+if SYSTEM == "rocm":
+    from vllm.model_executor.layers.fused_moe import fused_moe
+elif SYSTEM != "ipex":
+    from moe_kernels.fused_moe import fused_moe
+
+
+class UnquantizedSparseMoELayer(nn.Module):
+    def __init__(
+        self,
+        *,
+        n_expert_group: Optional[int],
+        n_experts: int,
+        prefix: str,
+        renormalize: bool,
+        topk: int,
+        topk_group: Optional[int],
+        weights: Weights,
+        gate_proj_name: str = "gate_proj",
+        up_proj_name: str = "up_proj",
+        down_proj_name: str = "down_proj",
+    ):
+        super().__init__()
+
+        assert (n_expert_group is None) == (
+            topk_group is None
+        ), "n_expert_group and topk_group must both be None or have some value"
+
+        self.n_expert_group = n_expert_group
+        self.topk = topk
+        self.topk_group = topk_group
+        self.renormalize = renormalize
+
+        self.gate_up_proj = _load_expert_multi_weights_col(
+            prefix=prefix,
+            n_experts=n_experts,
+            gate_proj_name=gate_proj_name,
+            up_proj_name=up_proj_name,
+            weights=weights,
+        )
+
+        self.down_proj = _load_expert_weights_row(
+            prefix=prefix,
+            n_experts=n_experts,
+            name=down_proj_name,
+            weights=weights,
+        )
+
+    def forward(self, x: torch.Tensor, *, gating_output: torch.Tensor) -> torch.Tensor:
+        if SYSTEM == "rocm":
+            return fused_moe(
+                x,
+                self.gate_up_proj,
+                self.down_proj,
+                gating_output,
+                self.topk,
+                renormalize=self.renormalize,
+                inplace=True,
+            )
+
+        return fused_moe(
+            x,
+            w1=self.gate_up_proj,
+            w2=self.down_proj,
+            gating_output=gating_output,
+            topk=self.topk,
+            renormalize=self.renormalize,
+            inplace=True,
+            use_grouped_topk=self.n_expert_group is not None,
+            num_expert_group=self.n_expert_group,
+            topk_group=self.topk_group,
+        )
+
+
+def _load_expert_multi_weights_col(
+    *,
+    prefix: str,
+    n_experts: int,
+    gate_proj_name: str,
+    up_proj_name: str,
+    weights: Weights,
+) -> torch.Tensor:
+    all_weight = None
+    for i in range(n_experts):
+        weight = weights.get_multi_weights_col(
+            [f"{prefix}.{i}.{gate_proj_name}", f"{prefix}.{i}.{up_proj_name}"], 0
+        )
+
+        assert isinstance(weight, UnquantizedWeight)
+
+        if all_weight is None:
+            all_weight = torch.empty(
+                (n_experts,) + weight.weight.shape,
+                dtype=weight.weight.dtype,
+                device=weight.weight.device,
+            )
+
+        all_weight[i] = weight.weight
+
+    assert all_weight is not None
+
+    return all_weight
+
+
+def _load_expert_weights_row(
+    *,
+    prefix: str,
+    n_experts: int,
+    name: str,
+    weights: Weights,
+) -> torch.Tensor:
+    all_weight = None
+    for i in range(n_experts):
+        weight = weights.get_weights_row(
+            f"{prefix}.{i}.{name}",
+        )
+
+        assert isinstance(weight, UnquantizedWeight)
+
+        if all_weight is None:
+            all_weight = torch.empty(
+                (n_experts,) + weight.weight.shape,
+                dtype=weight.weight.dtype,
+                device=weight.weight.device,
+            )
+
+        all_weight[i] = weight.weight
+
+    assert all_weight is not None
+
+    return all_weight
diff --git a/server/text_generation_server/layers/rotary.py b/server/text_generation_server/layers/rotary.py
index 648d28ab..a2076bb2 100644
--- a/server/text_generation_server/layers/rotary.py
+++ b/server/text_generation_server/layers/rotary.py
@@ -1,15 +1,14 @@
 import os
+import math
 import torch
 from torch import nn
-
 from text_generation_server.utils.import_utils import SYSTEM
 
 if SYSTEM == "cuda":
-    from flash_attn.layers.rotary import RotaryEmbedding
     import rotary_emb
 elif SYSTEM == "rocm":
     from vllm._C import ops
-elif SYSTEM == "xpu":
+elif SYSTEM == "ipex":
     import intel_extension_for_pytorch as ipex
 
 
@@ -69,7 +68,7 @@ class PositionRotaryEmbedding(nn.Module):
 
             # Inplace operation, updating query and key.
             ops.rotary_embedding(query, key, head_size, cos, sin, True)
-        elif SYSTEM == "xpu":
+        elif SYSTEM == "ipex":
             ipex.llm.functional.rotary_embedding(
                 query, key, sin, cos, query.size(-1), True
             )
@@ -84,9 +83,13 @@ class PositionRotaryEmbedding(nn.Module):
         scaling_factor = None
         rope_scaling = _get_rope_config(config)
         if rope_scaling is not None:
-            if rope_scaling["type"] == "linear":
+            # `rope_type` is now standard in transformers, but some existing models
+            # have `type` instead.
+            rope_type = rope_scaling.get("rope_type", rope_scaling.get("type", None))
+
+            if rope_type == "linear":
                 pass
-            elif rope_scaling["type"] == "dynamic":
+            elif rope_type == "dynamic":
                 scaling_factor = rope_scaling["factor"]
                 return DynamicPositionRotaryEmbedding(
                     dim=dim,
@@ -95,22 +98,39 @@ class PositionRotaryEmbedding(nn.Module):
                     device=inv_freq.device,
                     scaling_factor=scaling_factor,
                 )
-            elif rope_scaling["type"] == "yarn":
+            elif rope_type == "llama3":
+                inv_freq = apply_llama3_scaling(
+                    inv_freq,
+                    scaling_factor=rope_scaling["factor"],
+                    low_freq_factor=rope_scaling["low_freq_factor"],
+                    high_freq_factor=rope_scaling["high_freq_factor"],
+                    original_max_position_embeddings=rope_scaling[
+                        "original_max_position_embeddings"
+                    ],
+                )
+
+                return cls(inv_freq, scaling_factor)
+
+            elif rope_type == "yarn":
                 scaling_factor = rope_scaling["factor"]
+                mscale = rope_scaling.get("mscale", 1.0)
+                mscale_all_dim = rope_scaling.get("mscale_all_dim", 0.0)
                 return YarnPositionRotaryEmbedding(
                     dim=2 * inv_freq.shape[0],
                     max_position_embeddings=rope_scaling[
                         "original_max_position_embeddings"
                     ],
-                    base=10000.0,
+                    base=base,
                     device=inv_freq.device,
                     scaling_factor=scaling_factor,
                     extrapolation_factor=1,
                     attn_factor=1,
                     beta_fast=32,
                     beta_slow=1,
+                    mscale=mscale,
+                    mscale_all_dim=mscale_all_dim,
                 )
-            elif rope_scaling["type"] == "su":
+            elif rope_type in ["su", "longrope"]:
                 short_factor = torch.tensor(
                     rope_scaling["short_factor"], dtype=torch.float32, device=device
                 )
@@ -146,6 +166,20 @@ class PositionRotaryEmbedding(nn.Module):
                         1 + math.log(scale) / math.log(original_max_position_embeddings)
                     )
 
+                # if short_mscale and long_mscale are provided we need to scale the freqs
+                # using the Phi3LongRoPEScaledRotaryEmbedding
+                if ("short_mscale" in rope_scaling) and ("long_mscale" in rope_scaling):
+                    short_mscale = rope_scaling["short_mscale"]
+                    long_mscale = rope_scaling["long_mscale"]
+                    return Phi3LongRoPEScaledRotaryEmbedding(
+                        short_inv_freq=short_inv_freq,
+                        long_inv_freq=long_inv_freq,
+                        max_position_embeddings=config.max_position_embeddings,
+                        short_mscale=short_mscale,
+                        long_mscale=long_mscale,
+                        original_max_position_embeddings=original_max_position_embeddings,
+                    )
+
                 return SuRotaryEmbedding(
                     short_inv_freq=short_inv_freq,
                     long_inv_freq=long_inv_freq,
@@ -181,6 +215,8 @@ class PositionRotaryEmbedding(nn.Module):
                     scaling_factor=scaling_factor,
                 )
             elif rope_scaling["type"] == "yarn":
+                mscale = rope_scaling.get("mscale", 1.0)
+                mscale_all_dim = rope_scaling.get("mscale_all_dim", 0.0)
                 return YarnPositionRotaryEmbedding(
                     dim=2 * inv_freq.shape[0],
                     max_position_embeddings=rope_scaling[
@@ -193,6 +229,8 @@ class PositionRotaryEmbedding(nn.Module):
                     attn_factor=1,
                     beta_fast=32,
                     beta_slow=1,
+                    mscale=mscale,
+                    mscale_all_dim=mscale_all_dim,
                 )
             else:
                 raise NotImplementedError(
@@ -263,21 +301,81 @@ class SuRotaryEmbedding(PositionRotaryEmbedding):
         # or if we're on a new device (possibly due to tracing for instance)
         if (
             seqlen > self._seq_len_cached
+            or self._cos_cached is None
             or self._cos_cached.device != device
             or self._cos_cached.dtype != dtype
         ):
             self._seq_len_cached = seqlen
-            if seqlen > self.original_max_position_embeddings:
-                inv_freq = self.long_inv_freq
-            else:
-                inv_freq = self.short_inv_freq
-            t = torch.arange(seqlen, device=device, dtype=inv_freq.dtype)
-            if self.scaling_factor is not None:
-                t /= self.scaling_factor
-            # Don't do einsum, it converts fp32 to fp16
-            # freqs = torch.einsum("i,j->ij", t, self.inv_freq)
 
-            freqs = torch.outer(t, inv_freq.to(device=t.device))
+            t = torch.arange(seqlen, device=device, dtype=self.short_inv_freq.dtype)
+            short_freqs = torch.outer(
+                t[: self.original_max_position_embeddings],
+                self.short_inv_freq.to(device=t.device),
+            )
+            long_freqs = torch.outer(
+                t[self.original_max_position_embeddings :],
+                self.long_inv_freq.to(device=t.device),
+            )
+
+            freqs = torch.cat([short_freqs, long_freqs])
+
+            self._cos_cached = (torch.cos(freqs) * self.scaling_factor).to(dtype)
+            self._sin_cached = (torch.sin(freqs) * self.scaling_factor).to(dtype)
+
+
+class Phi3LongRoPEScaledRotaryEmbedding(PositionRotaryEmbedding):
+    def __init__(
+        self,
+        short_inv_freq: torch.Tensor,
+        long_inv_freq: torch.Tensor,
+        max_position_embeddings: int,
+        short_mscale: float,
+        long_mscale: float,
+        original_max_position_embeddings: int,
+    ):
+        super(PositionRotaryEmbedding, self).__init__()
+        self.short_inv_freq = short_inv_freq
+        self.long_inv_freq = long_inv_freq
+        self.max_position_embeddings = max_position_embeddings
+        self.short_mscale = short_mscale
+        self.long_mscale = long_mscale
+        self.original_max_position_embeddings = original_max_position_embeddings
+
+        # cache
+        self._seq_len_cached = 0
+        self._cos_cached = None
+        self._sin_cached = None
+        self._cos_k_cached = None
+        self._sin_k_cached = None
+        self.dynamic_args = None
+
+    def _update_cos_sin_cache(self, dtype, device, seqlen):
+        if (
+            seqlen > self._seq_len_cached
+            or self._cos_cached is None
+            or self._cos_cached.device != device
+            or self._cos_cached.dtype != dtype
+        ):
+            self._seq_len_cached = seqlen
+            t = torch.arange(seqlen, device=device, dtype=self.short_inv_freq.dtype)
+
+            short_freqs = torch.outer(
+                t[: self.original_max_position_embeddings],
+                self.short_inv_freq.to(device=t.device),
+            )
+
+            long_freqs = torch.outer(
+                t[self.original_max_position_embeddings :],
+                self.long_inv_freq.to(device=t.device),
+            )
+
+            short_freqs = short_freqs * self.short_mscale
+            long_freqs = long_freqs * self.long_mscale
+
+            freqs = torch.empty((seqlen, short_freqs.shape[1]), device=device)
+            freqs[: self.original_max_position_embeddings] = short_freqs
+            freqs[self.original_max_position_embeddings :] = long_freqs
+
             self._cos_cached = torch.cos(freqs).to(dtype)
             self._sin_cached = torch.sin(freqs).to(dtype)
 
@@ -316,10 +414,6 @@ class DynamicPositionRotaryEmbedding(PositionRotaryEmbedding):
             self._sin_cached = torch.sin(freqs).to(dtype)
 
 
-# Inverse dim formula to find dim based on number of rotations
-import math
-
-
 def find_correction_dim(num_rotations, dim, base=10000, max_position_embeddings=2048):
     return (dim * math.log(max_position_embeddings / (num_rotations * 2 * math.pi))) / (
         2 * math.log(base)
@@ -344,10 +438,10 @@ def linear_ramp_mask(min, max, dim):
     return ramp_func
 
 
-def get_mscale(scale=1):
+def get_mscale(scale: float = 1.0, mscale: float = 1.0):
     if scale <= 1:
         return 1.0
-    return 0.1 * math.log(scale) + 1.0
+    return 0.1 * mscale * math.log(scale) + 1.0
 
 
 class YarnPositionRotaryEmbedding(PositionRotaryEmbedding):
@@ -363,6 +457,8 @@ class YarnPositionRotaryEmbedding(PositionRotaryEmbedding):
         attn_factor,
         beta_fast,
         beta_slow,
+        mscale: float,
+        mscale_all_dim: float,
     ):
         inv_freq = _create_inv_freq(dim, base, device)
         super().__init__(inv_freq, scaling_factor)
@@ -373,8 +469,12 @@ class YarnPositionRotaryEmbedding(PositionRotaryEmbedding):
         self.attn_factor = attn_factor
         self.beta_fast = beta_fast
         self.beta_slow = beta_slow
+        self.mscale_all_dim = mscale_all_dim
+        self.scaling_factor = scaling_factor
         self.mscale = float(
-            get_mscale(self.scaling_factor) * self.attn_factor
+            get_mscale(self.scaling_factor, mscale)
+            / get_mscale(self.scaling_factor, mscale_all_dim)
+            * self.attn_factor
         )  # Get n-d magnitude scaling corrected for interpolation
 
     def _update_cos_sin_cache(self, dtype, device, seqlen):
@@ -385,7 +485,7 @@ class YarnPositionRotaryEmbedding(PositionRotaryEmbedding):
             or self._cos_cached.device != device
             or self._cos_cached.dtype != dtype
         ):
-            if seqlen > self.max_position_embeddings:
+            if seqlen > self.max_position_embeddings or True:
                 inv_freq_extrapolation = _create_inv_freq(
                     self.dim, self.base, self.inv_freq.device
                 )
@@ -398,6 +498,7 @@ class YarnPositionRotaryEmbedding(PositionRotaryEmbedding):
                     self.base,
                     self.max_position_embeddings,
                 )
+
                 inv_freq_mask = (
                     1 - linear_ramp_mask(low, high, self.dim // 2).float().to(device)
                 ) * self.extrapolation_factor  # Get n-d rotational scaling corrected for extrapolation
@@ -407,9 +508,6 @@ class YarnPositionRotaryEmbedding(PositionRotaryEmbedding):
                 )
 
                 self.inv_freq = inv_freq
-                self.mscale = float(
-                    get_mscale(self.scaling_factor) * self.attn_factor
-                )  # Get n-d magnitude scaling corrected for interpolation
 
             self._seq_len_cached = seqlen
             t = torch.arange(seqlen, device=device, dtype=self.inv_freq.dtype)
@@ -419,3 +517,32 @@ class YarnPositionRotaryEmbedding(PositionRotaryEmbedding):
             freqs = torch.outer(t, self.inv_freq.to(device=t.device))
             self._cos_cached = (torch.cos(freqs) * self.mscale).to(dtype)
             self._sin_cached = (torch.sin(freqs) * self.mscale).to(dtype)
+
+
+def apply_llama3_scaling(
+    freqs: torch.Tensor,
+    *,
+    scaling_factor: int,
+    low_freq_factor: int,
+    high_freq_factor: int,
+    original_max_position_embeddings: int,
+):
+    low_freq_wavelen = original_max_position_embeddings / low_freq_factor
+    high_freq_wavelen = original_max_position_embeddings / high_freq_factor
+    new_freqs = []
+
+    for freq in freqs:
+        wavelen = 2 * math.pi / freq
+
+        if wavelen < high_freq_wavelen:
+            new_freqs.append(freq)
+        elif wavelen > low_freq_wavelen:
+            new_freqs.append(freq / scaling_factor)
+        else:
+            assert low_freq_wavelen != high_freq_wavelen
+            smooth = (original_max_position_embeddings / wavelen - low_freq_factor) / (
+                high_freq_factor - low_freq_factor
+            )
+            new_freqs.append((1 - smooth) * freq / scaling_factor + smooth * freq)
+
+    return torch.tensor(new_freqs, dtype=freqs.dtype, device=freqs.device)
diff --git a/server/text_generation_server/layers/speculative.py b/server/text_generation_server/layers/speculative.py
index 4b977a56..cf8469b5 100644
--- a/server/text_generation_server/layers/speculative.py
+++ b/server/text_generation_server/layers/speculative.py
@@ -33,7 +33,7 @@ class SpeculativeHead(torch.nn.Module):
             except KeyError:
                 try:
                     speculator = MedusaHeadV1.load(config, prefix, weights)
-                except:
+                except Exception:
                     speculator = MedusaHeadV2(config, prefix, weights)
             lm_head = None
         else:
diff --git a/server/text_generation_server/layers/tensor_parallel.py b/server/text_generation_server/layers/tensor_parallel.py
index 34b9c51e..13f12ef1 100644
--- a/server/text_generation_server/layers/tensor_parallel.py
+++ b/server/text_generation_server/layers/tensor_parallel.py
@@ -1,7 +1,30 @@
 import torch
 from torch.nn import functional as F
-from typing import List
+from typing import Iterable, List
 from text_generation_server.layers.linear import get_linear, FastLinear
+from text_generation_server.utils.import_utils import SYSTEM
+
+if SYSTEM == "ipex":
+    import intel_extension_for_pytorch as ipex
+
+
+class LayerConcat(torch.nn.Module):
+    """
+    Apply multiple layers to the input and concatenate their
+    outputs.
+    """
+
+    def __init__(self, layers: Iterable[torch.nn.Module], dim: int = -1):
+        """
+        `dim` is the dimension along which layer outputs are concatenated.
+        """
+        super().__init__()
+        self.layers = layers
+        self.dim = dim
+
+    def forward(self, x: torch.Tensor):
+        outputs = [layer(x) for layer in self.layers]
+        return torch.cat(outputs, self.dim)
 
 
 class SuperLayer(torch.nn.Module):
@@ -21,7 +44,16 @@ class TensorParallelHead(SuperLayer):
 
     @staticmethod
     def load(config, prefix: str, weights):
-        if weights.process_group.size() > 1:
+        if config.quantize == "exl2":
+            try:
+                # If the piece and LM head embeddings are shared, we have
+                # non-quantized weights...
+                weight = weights.get_tensor(f"{prefix}.weight")
+            except Exception:
+                # ...otherwise they are quantized.
+                weight = weights.get_weights_col(prefix)
+            should_gather = weights.process_group.size() > 1
+        elif weights.process_group.size() > 1:
             try:
                 weight = weights.get_sharded(f"{prefix}.weight", dim=0)
                 should_gather = True
@@ -34,13 +66,8 @@ class TensorParallelHead(SuperLayer):
             weight = weights.get_tensor(f"{prefix}.weight")
             should_gather = False
 
-        # GPTQ,AWQ,EETQ don't quantize heads (nor embeddings)
-        if config.quantize in ["gptq", "awq", "eetq"]:
-            quantize = None
-        else:
-            quantize = config.quantize
         return TensorParallelHead(
-            get_linear(weight, bias=None, quantize=quantize),
+            get_linear(weight, bias=None),
             process_group=weights.process_group,
             should_gather=should_gather,
         )
@@ -63,10 +90,14 @@ class TensorParallelHead(SuperLayer):
                 local_out = gather_input.T
 
             torch.mm(input, self.linear.weight.T, out=local_out)
-
-            torch.distributed.all_gather_into_tensor(
-                world_out, gather_input, group=self.process_group
-            )
+            if SYSTEM == "ipex":
+                ipex.distributed.all_gather_into_tensor(
+                    world_out, gather_input, group=self.process_group
+                )
+            else:
+                torch.distributed.all_gather_into_tensor(
+                    world_out, gather_input, group=self.process_group
+                )
 
             if input.shape[0] == 1:
                 return world_out
@@ -76,7 +107,10 @@ class TensorParallelHead(SuperLayer):
         world_output = [
             torch.empty_like(output) for _ in range(self.process_group.size())
         ]
-        torch.distributed.all_gather(world_output, output, group=self.process_group)
+        if SYSTEM == "ipex":
+            ipex.distributed.all_gather(world_output, output, group=self.process_group)
+        else:
+            torch.distributed.all_gather(world_output, output, group=self.process_group)
         world_output = torch.cat(world_output, dim=-1)
         return world_output
 
@@ -85,43 +119,64 @@ class TensorParallelColumnLinear(SuperLayer):
     @classmethod
     def load_gate_up(cls, config, prefix: str, weights, bias: bool):
         """Specific method when the QKV was joined after the fact"""
-        weight = weights.get_weights_col_packed_gate_up(
-            prefix, quantize=config.quantize
-        )
+        weight = weights.get_weights_col_packed_gate_up(prefix)
         if bias:
             raise NotImplementedError("packed_gate_up only implemented without bias")
         else:
             bias = None
-        linear = get_linear(weight, bias, config.quantize)
+        linear = get_linear(weight, bias)
         return cls(linear)
 
     @classmethod
-    def load_qkv(cls, config, prefix: str, weights, bias: bool):
+    def load_qkv(
+        cls,
+        config,
+        prefix: str,
+        weights,
+        bias: bool,
+        num_heads: int,
+        num_key_value_heads: int,
+    ):
         """Specific method when the QKV was joined after the fact"""
-        weight = weights.get_weights_col_packed_qkv(prefix, quantize=config.quantize)
+        weight = weights.get_weights_col_packed_qkv(
+            prefix,
+            num_heads=num_heads,
+            num_key_value_heads=num_key_value_heads,
+        )
         if bias:
             raise NotImplementedError("packed_qkv only implemented for baichuan")
         else:
             bias = None
-        linear = get_linear(weight, bias, config.quantize)
+        linear = get_linear(weight, bias)
         return cls(linear)
 
     @classmethod
     def load(cls, config, prefix: str, weights, bias: bool):
-        return cls.load_multi(config, [prefix], weights, bias, dim=0)
+        weight = weights.get_weights_col(prefix)
+        if bias:
+            bias = weights.get_sharded(f"{prefix}.bias", dim=0)
+        else:
+            bias = None
+        linear = get_linear(weight, bias)
+        return cls(linear)
 
     @classmethod
     def load_multi(cls, config, prefixes: List[str], weights, bias: bool, dim: int):
-        weight = weights.get_multi_weights_col(
-            prefixes, quantize=config.quantize, dim=dim
-        )
-
-        if bias:
-            b = [weights.get_sharded(f"{p}.bias", dim=0) for p in prefixes]
-            bias = torch.cat(b, dim=dim)
+        if config.quantize == "exl2":
+            linears = []
+            for prefix in prefixes:
+                weight = weights.get_weights_col(prefix)
+                b = weights.get_tensor(f"{prefix}.bias") if bias else None
+                linears.append(get_linear(weight, b))
+            linear = LayerConcat(linears)
         else:
-            bias = None
-        linear = get_linear(weight, bias, config.quantize)
+            weight = weights.get_multi_weights_col(prefixes, dim=dim)
+            if bias:
+                b = [weights.get_sharded(f"{p}.bias", dim=0) for p in prefixes]
+                bias = torch.cat(b, dim=dim)
+            else:
+                bias = None
+            linear = get_linear(weight, bias)
         return cls(linear)
 
 
@@ -132,7 +187,7 @@ class TensorParallelRowLinear(SuperLayer):
 
     @classmethod
     def load(cls, config, prefix: str, weights, bias: bool):
-        weight = weights.get_multi_weights_row(prefix, quantize=config.quantize)
+        weight = weights.get_weights_row(prefix)
 
         if bias and weights.process_group.rank() == 0:
             # Rank is only on the first rank process
@@ -140,14 +195,17 @@ class TensorParallelRowLinear(SuperLayer):
         else:
             bias = None
         return cls(
-            get_linear(weight, bias, config.quantize),
+            get_linear(weight, bias),
             process_group=weights.process_group,
         )
 
     def forward(self, input: torch.Tensor, reduce: bool = True) -> torch.Tensor:
         out = super().forward(input)
         if self.process_group.size() > 1 and reduce:
-            torch.distributed.all_reduce(out, group=self.process_group)
+            if SYSTEM == "ipex":
+                ipex.distributed.all_reduce(out, group=self.process_group)
+            else:
+                torch.distributed.all_reduce(out, group=self.process_group)
         return out
 
 
@@ -184,5 +242,8 @@ class TensorParallelEmbedding(torch.nn.Module):
         )
         out = torch.nn.functional.embedding(input, self.weight)
         if self.reduce and self.process_group.size() > 1:
-            torch.distributed.all_reduce(out, group=self.process_group)
+            if SYSTEM == "ipex":
+                ipex.distributed.all_reduce(out, group=self.process_group)
+            else:
+                torch.distributed.all_reduce(out, group=self.process_group)
         return out
diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py
index 30bd8f6c..dc2cb21c 100644
--- a/server/text_generation_server/models/__init__.py
+++ b/server/text_generation_server/models/__init__.py
@@ -7,7 +7,7 @@ from transformers.models.auto import modeling_auto
 from huggingface_hub import hf_hub_download, HfApi
 from typing import Optional
 from pathlib import Path
-
+from typing import Optional, List, Dict
 # Needed to properly setup habana_frameworks
 import text_generation_server.habana_quantization_env as hq_env
 
@@ -17,10 +17,20 @@ from text_generation_server.models.causal_lm import CausalLM
 from text_generation_server.models.bloom import BLOOM
 from text_generation_server.models.starcoder import StarCoder
 from text_generation_server.models.vlm_causal_lm import VlmCausalLM
+#from text_generation_server.models.mllama_causal_lm import MllamaCausalLM
 from text_generation_server.models.custom_modeling.llava_next import (
     LlavaNextForConditionalGeneration,
 )
-
+# from text_generation_server.models.mllama_causal_lm import MllamaCausalLMBatch
+# from text_generation_server.models.custom_modeling.mllama import (
+#     MllamaForConditionalGeneration,
+# )
+from text_generation_server.utils.adapter import (
+    AdapterParameters,
+    build_layer_weight_lookup,
+    load_and_merge_adapters,
+    AdapterInfo,
+)
 
 
 from optimum.habana.transformers.modeling_utils import adapt_transformers_to_gaudi
@@ -32,10 +42,14 @@ torch.set_grad_enabled(False)
 
 def get_model(
     model_id: str,
+    lora_adapter_ids: Optional[List[str]],
     revision: Optional[str],
+    sharded: bool,
+    quantize: Optional[str],
     speculate: Optional[int],
     dtype: Optional[torch.dtype],
     trust_remote_code: bool,
+    max_input_tokens: int,
 ) -> Model:
     adapt_transformers_to_gaudi()
 
@@ -136,6 +150,11 @@ def get_model(
                     revision=mlp_revision,
                     filename=filename,
                 )
+            speculator_dir_path = Path(mlp_speculator_config).parent
+            # if these are downloaded, they get converted to safetensors
+            filenames.extend(
+                [p for p in os.listdir(speculator_dir_path) if p.endswith(extension)]
+            )
             speculator = {
                 "path": Path(mlp_speculator_config).parent,
                 "model_paths": filenames,
@@ -155,12 +174,12 @@ def get_model(
     model_type = config_dict["model_type"]
 
     if model_type == "gpt_bigcode":
-        return StarCoder(model_id, revision, dtype)
+        return StarCoder(model_id=model_id, revision=revision, dtype=dtype)
 
     if model_type == "bloom":
         return BLOOM(
-            model_id,
-            revision,
+            model_id=model_id,
+            revision=revision,
             speculator=speculator,
             dtype=dtype,
             trust_remote_code=trust_remote_code,
@@ -181,9 +200,124 @@ def get_model(
         return CausalLM(
             model_id,
             revision,
+            quantize=quantize,
             speculator=speculator,
             dtype=dtype,
             trust_remote_code=trust_remote_code,
         )
 
     raise ValueError(f"Unsupported model type {model_type}")
+
+
+# get_model_with_lora_adapters wraps the internal get_model function and adds support for loading adapters
+# this provides a post model loading hook to load adapters into the model after the model has been loaded
+def get_model_with_lora_adapters(
+    model_id: str,
+    lora_adapters: Optional[List[AdapterInfo]],
+    revision: Optional[str],
+    sharded: bool,
+    quantize: Optional[str],
+    speculate: Optional[int],
+    dtype: Optional[torch.dtype],
+    trust_remote_code: bool,
+    max_input_tokens: int,
+    adapter_to_index: Dict[str, int],
+):
+    lora_adapter_ids = [adapter.id for adapter in lora_adapters]
+    model = get_model(
+        model_id,
+        lora_adapter_ids,
+        revision,
+        sharded,
+        quantize,
+        speculate,
+        dtype,
+        trust_remote_code,
+        max_input_tokens,
+    )
+
+    if len(lora_adapters) > 0:
+        target_to_layer = build_layer_weight_lookup(model.model)
+
+        for index, adapter in enumerate(lora_adapters):
+            # The AdapterParameters object allows for merging multiple adapters into a single adapter.
+            # At the moment, we only support loading a single adapter into the model, but we keep the
+            # AdapterParameters object for easier extension in the future.
+            adapter_parameters = AdapterParameters(
+                adapter_info=[adapter],
+                # when merging multiple adapters we can weight them differently
+                # if this is not set, all adapters will be weighted equally
+                # see: text_generation_server.utils.merges.strategies for impl
+                weights=None,
+                merge_strategy=0,
+                density=1.0,
+                majority_sign_method=0,
+            )
+
+            adapter_index = index + 1
+            adapter_to_index[adapter.id] = adapter_index
+
+            logger.info(
+                f"Loading adapter weights into model: {','.join([adapter.id for adapter in adapter_parameters.adapter_info])}"
+            )
+            weight_names = tuple([v[0] for v in target_to_layer.values()])
+            (
+                module_map,
+                adapter_config,
+                adapter_weight_names,
+                adapter_tokenizer,
+            ) = load_and_merge_adapters(
+                model.model_id,
+                adapter_parameters,
+                adapter_index,
+                weight_names,
+                False,
+            )
+
+            unused_weight_names = adapter_weight_names.copy()
+
+            adapter_layers = [
+                "q_proj",
+                "k_proj",
+                "v_proj",
+                "o_proj",
+                "gate_proj",
+                "up_proj",
+                "down_proj",
+                "qkv_proj",
+            ]
+
+            for layer_name in adapter_layers:
+                nlayers = (
+                    1 if layer_name == "lm_head" else len(model.model.model.layers)
+                )
+                adapter_weights = LoraWeights.prepare_weights(
+                    config=adapter_config,
+                    module_map=module_map,
+                    layer_type=layer_name,
+                    unused_weight_names=unused_weight_names,
+                    nlayers=nlayers,
+                    dtype=model.dtype,
+                    world_size=model.world_size,
+                    process_group=model.process_group,
+                    target_to_layer=target_to_layer,
+                )
+
+                if adapter_weights is None:
+                    continue
+
+                model.layer_to_adapter_weights[layer_name].add_adapter(
+                    adapter_index, adapter_weights
+                )
+
+            if len(unused_weight_names) > 0:
+                logger.warning(
+                    f"{','.join([a.id for a in lora_adapters])} unused adapter weights: {unused_weight_names}"
+                )
+
+            if adapter_tokenizer is not None:
+                model.tokenizers.add_tokenizer(adapter_index, adapter_tokenizer)
+
+            model.loaded_adapters.add(adapter_index)
+
+    return model
diff --git a/server/text_generation_server/models/cache_manager.py b/server/text_generation_server/models/cache_manager.py
deleted file mode 100644
index c7705fe8..00000000
--- a/server/text_generation_server/models/cache_manager.py
+++ /dev/null
@@ -1,140 +0,0 @@
-import math
-import torch
-
-from typing import Optional, List, Tuple
-from text_generation_server.utils.import_utils import SYSTEM
-
-BLOCK_SIZE: int = 16
-# Will be set in warmup
-CACHE_MANAGER: Optional["CacheManager"] = None
-
-
-class CacheManager:
-    def __init__(
-        self,
-        num_blocks: int,
-        num_layers: int,
-        num_heads: int,
-        head_size: int,
-        repeat_slots: bool,
-        dtype: torch.dtype,
-        device: torch.device,
-    ):
-        self.block_size = BLOCK_SIZE
-        self.num_blocks = num_blocks
-        self.repeat_slots = repeat_slots
-
-        element_size = torch.tensor([], dtype=dtype).element_size()
-        if SYSTEM == "xpu":
-            x = 1
-        else:
-            x = self.block_size // element_size
-
-        self.kv_cache = [
-            (
-                torch.empty(
-                    (num_blocks, num_heads, head_size // x, self.block_size, x),
-                    dtype=dtype,
-                    device=device,
-                ),
-                torch.empty(
-                    (num_blocks, num_heads, head_size, self.block_size),
-                    dtype=dtype,
-                    device=device,
-                ),
-            )
-            for _ in range(num_layers)
-        ]
-        self.free_block_mask = torch.ones(num_blocks, dtype=torch.int32, device="cpu")
-        self.slots = torch.arange(
-            0, num_blocks * self.block_size, dtype=torch.int64
-        ).view(num_blocks, self.block_size)
-
-    def allocate(
-        self,
-        needed_blocks_slots: List[Tuple[int, int]],
-        blocks: int,
-        max_blocks: int,
-        device: torch.device,
-    ):
-        # Get free blocks indices by finding values in mask that are not set to 0
-        free_block_indices = self.free_block_mask.nonzero()
-        if blocks > len(free_block_indices):
-            raise RuntimeError(
-                f"Out of available cache blocks: asked {blocks}, only {len(free_block_indices)} free blocks"
-            )
-
-        # Slice by the number of required blocks
-        block_indices = free_block_indices[:blocks]
-        block_indices = block_indices.flatten()
-
-        # Padded block tables
-        block_tables_tensor = torch.zeros(
-            (len(needed_blocks_slots), max_blocks), dtype=torch.int32
-        )
-
-        # Allocate paged attention blocks
-        cumulative_blocks = 0
-        slots = []
-        block_tables = []
-        for i, (needed_blocks, needed_slots) in enumerate(needed_blocks_slots):
-            # Get allocated blocks for this sequence
-            allocated_blocks = block_indices[
-                cumulative_blocks : cumulative_blocks + needed_blocks
-            ]
-            # Get slots for the allocated blocks
-            all_slots = self.slots[allocated_blocks].flatten()
-
-            # Repeat slots in the case of context sliding window
-            if needed_slots > len(all_slots) and self.repeat_slots:
-                repeats = math.ceil(needed_slots / len(all_slots))
-                all_slots = all_slots.repeat(repeats)
-
-            allocated_slots = all_slots[:needed_slots]
-
-            slots.append(allocated_slots)
-            block_tables.append(allocated_blocks.tolist())
-            block_tables_tensor[i, :needed_blocks] = allocated_blocks
-            cumulative_blocks += needed_blocks
-
-        block_tables = block_tables
-        block_tables_tensor = block_tables_tensor.to(device)
-        slots = torch.concat(slots).to(device)
-
-        # Allocate the required number of blocks by setting the mask to 0
-        self.free_block_mask[block_indices] = 0
-
-        return block_tables, block_tables_tensor, slots
-
-    def free(self, block_indices: Optional[List[int]]):
-        if block_indices is not None and block_indices:
-            # Reset mask
-            self.free_block_mask[block_indices] = 1
-
-
-def set_cache_manager(
-    num_blocks: int,
-    num_layers: int,
-    num_heads: int,
-    head_size: int,
-    repeat_slots: bool,
-    dtype: torch.dtype,
-    device: torch.device,
-) -> CacheManager:
-    global CACHE_MANAGER
-    if CACHE_MANAGER is not None:
-        del CACHE_MANAGER
-        torch.cuda.empty_cache()
-
-    CACHE_MANAGER = CacheManager(
-        num_blocks, num_layers, num_heads, head_size, repeat_slots, dtype, device
-    )
-    return CACHE_MANAGER
-
-
-def get_cache_manager() -> CacheManager:
-    global CACHE_MANAGER
-    if CACHE_MANAGER is None:
-        raise RuntimeError("cache manager was not initialized")
-
-    return CACHE_MANAGER
diff --git a/server/text_generation_server/models/causal_lm.py b/server/text_generation_server/models/causal_lm.py
index 70b32e4a..120b140b 100644
--- a/server/text_generation_server/models/causal_lm.py
+++ b/server/text_generation_server/models/causal_lm.py
@@ -8,6 +8,7 @@ import math
 import os
 import tempfile
 import time
+import copy
 from typing import Dict, List, Optional, Tuple, Type
 
 import torch
@@ -19,6 +20,7 @@ import text_generation_server.habana_quantization_env as hq_env
 import habana_frameworks.torch as htorch
 from optimum.habana.utils import HabanaProfile
 from optimum.habana.transformers.generation import MODELS_OPTIMIZED_WITH_STATIC_SHAPES
+from text_generation_server.utils.chunks import concat_text_chunks
 from optimum.habana.checkpoint_utils import (
     get_repo_root,
     model_on_meta,
@@ -47,29 +49,26 @@ from text_generation_server.utils import (
     is_tokenizer_transparent,
     pad_next_token_chooser_parameters,
 )
+from optimum.habana.utils import get_hpu_memory_stats
 from text_generation_server.utils.debug import dbg_trace
 from text_generation_server.utils.speculate import get_speculate
 
 tracer = trace.get_tracer(__name__)
-
 MAX_TOTAL_TOKENS = int(os.environ.get('MAX_TOTAL_TOKENS', 2048))
-BATCH_BUCKET_SIZE = int(os.environ.get('BATCH_BUCKET_SIZE', 8))
-PAD_SEQUENCE_TO_MULTIPLE_OF = int(os.environ.get('PAD_SEQUENCE_TO_MULTIPLE_OF', 128))
-PREFILL_BATCH_BUCKET_SIZE = int(os.environ.get('PREFILL_BATCH_BUCKET_SIZE', 4))
+PAD_SEQUENCE_TO_MULTIPLE_OF = int(os.environ.get('PAD_SEQUENCE_TO_MULTIPLE_OF', 256))
 CHUNK_SIZES = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048]
 LAZY_MODE = int(os.environ.get('PT_HPU_LAZY_MODE', 1))
-
+BATCH_BUCKET_SIZE= int(os.environ.get('BATCH_BUCKET_SIZE', 8))
+PREFILL_BATCH_BUCKET_SIZE = int(os.environ.get('PREFILL_BATCH_BUCKET_SIZE', 2))
 
 def torch_compile_for_eager(func):
     if LAZY_MODE == 1:
         return func
     return torch.compile(func, backend="hpu_backend", options={"keep_input_mutations": True})
 
-
 def round_up(number, k):
     return (number + k - 1) // k * k
 
-
 def to_tensor_indices(indices, device):
     return torch.tensor(indices, dtype=torch.long, device=device)
 
@@ -250,6 +249,9 @@ class CausalLMBatch(Batch):
     past_key_values: Optional[List[Tuple]]
     merged_kv_cache: bool
 
+    # Lengths of all generations present in the batch
+    input_length: int
+
     # Generation helpers
     next_token_chooser: HeterogeneousNextTokenChooser
     top_n_tokens: List[int]
@@ -257,6 +259,7 @@ class CausalLMBatch(Batch):
 
     input_length: int
 
+    # Past metadata
     logits = None
     past = None
 
@@ -359,7 +362,9 @@ class CausalLMBatch(Batch):
             raise ValueError("KV cache not allocated! Cannot recombine before prefill!")
 
         total_requests = sum(len(b) for b in batches)
+        new_bs = total_requests
         new_bs = round_up(total_requests, BATCH_BUCKET_SIZE)
+
         batch_id = batches[0].batch_id
         device = batches[0].input_ids.device
 
@@ -463,21 +468,26 @@ class CausalLMBatch(Batch):
     ) -> "CausalLMBatch":
         dbg_trace('FROM_PB', f'num_reqs:{len(pb.requests)}')
         requests = [CausalLMRequest.from_pb(idx, req, tokenizer) for idx, req in enumerate(pb.requests)]
+        inputs = []
+        top_n_tokens = []
 
-        max_input_length = max(r.data.truncate for r in requests)
+        # Parse batch
+        max_truncation = 0
+        for i, r in enumerate(pb.requests):
+            inputs.append(concat_text_chunks(r.input_chunks.chunks))
+            top_n_tokens.append(r.top_n_tokens)
+            max_truncation = max(max_truncation, r.truncate)
+  
+        max_input_length = max_truncation
         if max_input_length < PAD_SEQUENCE_TO_MULTIPLE_OF:
              max_input_length = PAD_SEQUENCE_TO_MULTIPLE_OF
         max_new_tokens = max(r.stopping_criteria.max_new_tokens for r in requests)
 
-        # TODO: Add support for sparse batches
-        top_n_tokens = [r.top_n_tokens for r in pb.requests]
-        top_n_tokens_tensor = torch.tensor(top_n_tokens, device=device, dtype=torch.int64)
-
         # TODO: by tokenizing all inputs at once we loose information on actual input lengths
         # this means that we cannot shift inputs to the left after a long input sequence
         # was filtered out
         new_bs = round_up(len(requests), PREFILL_BATCH_BUCKET_SIZE)
-        missing_inputs = new_bs - len(requests)
+        missing_inputs = new_bs - len(inputs)
         dummy_inputs = ["?"] * missing_inputs
         parameters = [r.parameters for r in pb.requests]
         # append the dummy parameters for dummy request
@@ -490,17 +500,18 @@ class CausalLMBatch(Batch):
             tokenizer=tokenizer,
             quantization_enabled=hq_env.is_quantization_enabled,
         )
+
         tokenized_inputs = tokenizer(
-            [r.data.inputs for r in requests] + dummy_inputs,
+            inputs+dummy_inputs,
             return_tensors="pt",
             padding="longest",
             return_token_type_ids=False,
             truncation=True,
-            max_length=max_input_length,
+            max_length=max_truncation,
         )
 
         input_len = tokenized_inputs["input_ids"].shape[1]
-
+        # Round up sequence length
         bucket_size = max_input_length
         left_padding = max_input_length - input_len
         if input_len < max_input_length and PAD_SEQUENCE_TO_MULTIPLE_OF != 0:
@@ -525,8 +536,6 @@ class CausalLMBatch(Batch):
         all_input_ids = torch.nn.functional.pad(
             input_ids, (0, max_new_tokens), value=tokenizer.pad_token_id
         ).T.split(1, dim=1)
-
-        # New input length after left padding
         input_len = bucket_size
         for r in requests:
             r.input_length = input_len
@@ -539,8 +548,11 @@ class CausalLMBatch(Batch):
         position_ids = attention_mask.long().cumsum(-1) - 1
         position_ids.masked_fill_(attention_mask == 0, 1)
 
-        htorch.core.mark_step()
 
+        top_n_tokens_tensor = torch.tensor(
+            top_n_tokens, device=device, dtype=torch.int64
+        )
+        htorch.core.mark_step()
         return cls(
             batch_id=pb.id,
             requests=requests,
@@ -597,16 +609,24 @@ class CausalLM(Model):
     def __init__(
         self,
         model_id: str,
+        model_class: Optional[Type[torch.nn.Module]] = None,
         revision: Optional[str] = None,
+        quantize: Optional[str] = None,
         speculator: Optional[str] = None,
         dtype: Optional[torch.dtype] = None,
+        default_dtype=torch.float16,
         trust_remote_code: bool = False,
+        tokenizer_class=AutoTokenizer,
+        config_class=AutoConfig,
+        batch_class=CausalLMBatch,
+
     ):
 
         if speculator:
             raise RuntimeError("Speculator decoding is not enabled for AutoModel")
 
         self.prev_bs = 0
+        self.quantize = quantize
 
         # Create tokenizer
         tokenizer = AutoTokenizer.from_pretrained(
@@ -691,33 +711,33 @@ class CausalLM(Model):
             else:
                 tokenizer.add_special_tokens({"pad_token": "[PAD]"})
 
-        kwargs = {
+        self.kwargs = {
             "use_cache": True,
             "return_dict": True,
         }
 
+
         if model.config.model_type in ["llama", "mistral", "starcoder2", "qwen2", "falcon"]:
-
             if model.config.model_type not in ["falcon"]:
-                kwargs["attn_softmax_bf16"] = True
+                self.kwargs["attn_softmax_bf16"] = True
 
-            kwargs["trim_logits"] = True
+            self.kwargs["trim_logits"] = True
 
             if os.getenv("USE_FLASH_ATTENTION", "false").lower() == "true":
-                kwargs["use_flash_attention"] = True
+                self.kwargs["use_flash_attention"] = True
             if os.getenv("FLASH_ATTENTION_RECOMPUTE", "false").lower() == "true":
-                kwargs["flash_attention_recompute"] = True
+                self.kwargs["flash_attention_recompute"] = True
 
         self.speculate = get_speculate()
 
         super(CausalLM, self).__init__(
+            model_id=model_id,
             model=model,
             tokenizer=tokenizer,
             requires_padding=True,
             dtype=dtype,
             device=device,
             rank=rank,
-            kwargs=kwargs,
         )
 
         # Create profiler
@@ -849,6 +869,7 @@ class CausalLM(Model):
             kwargs["bypass_hpu_graphs"] = bypass_hpu_graph
 
         kwargs.update(self.kwargs)
+
         if past_key_values is not None and self.model.config.model_type not in ["gpt_bigcode"]:
             return self.model.forward(**kwargs)
         else:
@@ -914,6 +935,7 @@ class CausalLM(Model):
                         'top_token_ids': batch_top_token_ids[req_idx],
                         'top_token_logprobs': batch_top_token_logprobs[req_idx],
                         'grammar_state': batch.next_token_chooser.fsm_grammar_states[req.idx],
+
                     })
 
                 htorch.core.mark_step()
@@ -1147,35 +1169,104 @@ class CausalLM(Model):
         decode_ns = time.time_ns() - start_decode
         return generations, batch if not stopped else None, (forward_ns, decode_ns)
 
-    def warmup(self, batches: List[CausalLMBatch]) -> None:
-        def get_unfinished_requests(requests: List[CausalLMRequest]) -> List[int]:
-            return [
-                request.data.id for request in requests
-                if request.stopping_criteria.current_tokens < request.stopping_criteria.max_new_tokens
-            ]
+    def generate_warmup_batch(self, request, seq_len, batch_size):
+        batch = copy.deepcopy(request.batch)
+        for req in batch.requests:
+            req.truncate = seq_len
 
-        # prefill
-        _, prefill_batch, _ = self.generate_token([batches.pop(0)])
-        # decode
-        _, decode_batch, _ = self.generate_token([prefill_batch])
-        # shifts
-        self.shifting_warmup(decode_batch)
+        for i in range(len(batch.requests) - batch_size):
+            batch.requests.pop()
 
-        while len(batches) > 0:
-            # prefill
-            _, prefill_batch, _ = self.generate_token([batches.pop(0)])
-            # concatenate and decode
-            _, decode_batch, _ = self.generate_token([decode_batch, prefill_batch])
-            # filter finished requests
-            request_ids = get_unfinished_requests(decode_batch.requests)
-            if len(request_ids) < len(decode_batch.requests):
-                decode_batch = decode_batch.filter(request_ids)
+        return self.batch_type.from_pb(batch, self.tokenizer, self.dtype, self.device)
 
-    def shifting_warmup(self, batch: CausalLMBatch) -> None:
-        chunk_sizes = CHUNK_SIZES.copy()
-        chunk_sizes.extend([-chunk for chunk in chunk_sizes])
 
-        for chunk in chunk_sizes:
-            batch.merge_kv_cache_if_needed(batch.batch_size, chunk)
-            batch.realign(batch.batch_size, chunk, 0)
-            batch.split_kv_cache_if_needed(True)
+    def warmup(self, request) -> None:
+        MAX_TOTAL_TOKENS = request.max_total_tokens
+        MAX_BATCH_TOTAL_TOKENS = request.max_batch_total_tokens
+        batch = self.batch_type.from_pb(request.batch, self.tokenizer, self.dtype, self.device)
+        max_prefill_batch_size = batch.input_ids.shape[0]
+        try:
+            # max prefill batch size warmup
+            _, prefill_batch, _ = self.generate_token([batch])
+        except:
+            raise RuntimeError(
+                f"Not enough memory to handle {len(batch.input_ids)} prefill tokens. "
+                f"You need to decrease `--max-batch-prefill-tokens`"
+            )
+
+        del prefill_batch
+
+        # Warmup prefill batch_size
+        max_input_length =  request.max_input_length
+        prefill_batch_size_list = [batch for batch in range(BATCH_BUCKET_SIZE, max_prefill_batch_size, BATCH_BUCKET_SIZE)]
+        prefill_batch_size_list.append(max_prefill_batch_size)
+        prefill_seqlen_list = [seq for seq in range(PAD_SEQUENCE_TO_MULTIPLE_OF, max_input_length, PAD_SEQUENCE_TO_MULTIPLE_OF)]
+        prefill_seqlen_list.append(max_input_length)
+        prefill_batch_size_list.sort(reverse=True)
+        prefill_seqlen_list.sort(reverse=True)
+        try:
+            for batch_size in prefill_batch_size_list:
+                for seq_len in prefill_seqlen_list:
+                    batch = self.generate_warmup_batch(request, seq_len-1, batch_size)
+                    _, prefill_batch, _ = self.generate_token([batch])
+        except:
+            prefill_batch_size_list.sort()
+            prefill_seqlen_list.sort()
+            raise RuntimeError(
+                f"Not enough memory to run following prefill batch_size."
+                f"Prefill batch size list:{prefill_batch_size_list}"
+                f"Prefill sequence length list:{prefill_seqlen_list}"
+                f"You need to decrease `--max-batch-prefill-tokens`"
+            )
+        prefill_seqlen_list.sort()
+        prefill_batch_size_list.sort()
+        mem_stats = get_hpu_memory_stats(self.device)
+        logger.info(
+            f"\nFollowing prefill warmup successfully.\n"
+            f"Prefill batch size list:{prefill_batch_size_list}\n"
+            f"Prefill sequence length list:{prefill_seqlen_list}\n"
+            f"Memory stats: {mem_stats} "
+        )
+
+        #warmup decode batch size
+        max_decode_batch_size = math.floor(MAX_BATCH_TOTAL_TOKENS / MAX_TOTAL_TOKENS)
+        max_decode_batch_size = round_up(max_decode_batch_size, BATCH_BUCKET_SIZE)
+        decode_batch_size_list = [i for i in range(BATCH_BUCKET_SIZE, max_decode_batch_size, BATCH_BUCKET_SIZE)]
+        decode_batch_size_list.append(max_decode_batch_size)
+        decode_batch_size_list.sort(reverse=True)
+
+        try:
+            for batch_size in decode_batch_size_list:
+                batches= []
+                iters = math.floor(batch_size/max_prefill_batch_size)
+                for i in range(iters):
+                    batch = self.generate_warmup_batch(request,  PAD_SEQUENCE_TO_MULTIPLE_OF - 1, max_prefill_batch_size)
+                    _, prefill_batch, _ = self.generate_token([batch])
+                    batches.append(prefill_batch)
+
+                if batch_size % max_prefill_batch_size != 0:
+                    batch = self.generate_warmup_batch(request,  PAD_SEQUENCE_TO_MULTIPLE_OF - 1, batch_size % max_prefill_batch_size)
+                    _, prefill_batch, _ = self.generate_token([batch])
+                    batches.append(prefill_batch)
+
+                _, decode_batch, _ = self.generate_token(batches)
+                _, decode_batch, _ = self.generate_token([decode_batch])
+                del decode_batch
+                batches.clear()
+
+        except:
+                raise RuntimeError(
+                    f"Not enough memory to warmup decode batch_sizes({decode_batch_size_list})."
+                    f"You need to decrease `--max-batch-total-tokens`"
+                )
+
+        decode_batch_size_list.sort()
+        MAX_BATCH_TOTAL_TOKENS = MAX_TOTAL_TOKENS * decode_batch_size_list[-1]
+        mem_stats = get_hpu_memory_stats(self.device)
+        logger.info(
+            f"\nFollowing decode warmup successfully.\n"
+            f"Decode batch size list:{decode_batch_size_list}\n"
+            f"Memory stats: {mem_stats} "
+        )
+
+        return MAX_BATCH_TOTAL_TOKENS
\ No newline at end of file
diff --git a/server/text_generation_server/models/custom_modeling/bloom_modeling.py b/server/text_generation_server/models/custom_modeling/bloom_modeling.py
index 0d8a1b59..e2719fad 100644
--- a/server/text_generation_server/models/custom_modeling/bloom_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/bloom_modeling.py
@@ -816,7 +816,7 @@ class BloomModel(BloomPreTrainedModel):
 
 
 class BloomForCausalLM(BloomPreTrainedModel):
-    def __init__(self, config, weights):
+    def __init__(self, prefix: str, config, weights):
         super().__init__(config)
         self.transformer = BloomModel(config, weights)
 
@@ -908,7 +908,7 @@ class BloomForCausalLM(BloomPreTrainedModel):
         loss = None
 
         if not return_dict:
-            output = (lm_logits,) + transformer_outputs[1:]
+            output = (logits,) + transformer_outputs[1:]
             return ((loss,) + output) if loss is not None else output
 
         return (
diff --git a/server/text_generation_server/models/custom_modeling/clip.py b/server/text_generation_server/models/custom_modeling/clip.py
index 56618bf1..ab824da5 100644
--- a/server/text_generation_server/models/custom_modeling/clip.py
+++ b/server/text_generation_server/models/custom_modeling/clip.py
@@ -1,4 +1,4 @@
-from typing import Optional, Tuple, Union
+from typing import Optional, Tuple
 
 import torch
 from torch import nn
@@ -9,9 +9,7 @@ from transformers.modeling_attn_mask_utils import (
     _prepare_4d_attention_mask,
 )
 from transformers.modeling_outputs import (
-    BaseModelOutput,
     BaseModelOutputWithPooling,
-    ImageClassifierOutput,
 )
 from transformers import CLIPConfig, CLIPTextConfig, CLIPVisionConfig
 
@@ -446,11 +444,12 @@ class CLIPEncoder(nn.Module):
 
 
 class CLIPTextTransformer(nn.Module):
-    def __init__(self, config: CLIPTextConfig):
+    def __init__(self, prefix: str, config: CLIPTextConfig, weights=None):
         super().__init__()
         self.config = config
         embed_dim = config.hidden_size
         self.embeddings = CLIPTextEmbeddings(config)
+        # Initialize weights and apply final processing with `self.post_init()`
         self.encoder = CLIPEncoder(
             prefix=f"{prefix}.encoder", config=config, weights=weights
         )
@@ -505,7 +504,7 @@ class CLIPTextTransformer(nn.Module):
             # text_embeds.shape = [batch_size, sequence_length, transformer.width]
             # take features from the eot embedding (eot_token is the highest number in each sequence)
             # casting to torch.int for onnx compatibility: argmax doesn't support int64 inputs with opset 14
-            pooled_output = last_hidden_state[
+            last_hidden_state[
                 torch.arange(
                     last_hidden_state.shape[0], device=last_hidden_state.device
                 ),
@@ -515,7 +514,7 @@ class CLIPTextTransformer(nn.Module):
             ]
         else:
             # The config gets updated `eos_token_id` from PR #24773 (so the use of exta new tokens is possible)
-            pooled_output = last_hidden_state[
+            last_hidden_state[
                 torch.arange(
                     last_hidden_state.shape[0], device=last_hidden_state.device
                 ),
@@ -536,9 +535,9 @@ class CLIPTextModel(CLIPPreTrainedModel):
 
     _no_split_modules = ["CLIPTextEmbeddings", "CLIPEncoderLayer"]
 
-    def __init__(self, config: CLIPTextConfig):
+    def __init__(self, prefix, config: CLIPTextConfig):
         super().__init__(config)
-        self.text_model = CLIPTextTransformer(config)
+        self.text_model = CLIPTextTransformer(prefix, config)
         # Initialize weights and apply final processing
         self.post_init()
 
@@ -565,9 +564,6 @@ class CLIPTextModel(CLIPPreTrainedModel):
         >>> last_hidden_state = outputs.last_hidden_state
         >>> pooled_output = outputs.pooler_output  # pooled (EOS token) states
         ```"""
-        return_dict = (
-            return_dict if return_dict is not None else self.config.use_return_dict
-        )
 
         return self.text_model(
             input_ids=input_ids,
@@ -580,7 +576,6 @@ class CLIPVisionTransformer(nn.Module):
     def __init__(self, prefix, config: CLIPVisionConfig, weights):
         super().__init__()
         self.config = config
-        embed_dim = config.hidden_size
 
         self.embeddings = CLIPVisionEmbeddings(
             prefix=f"{prefix}.embeddings", config=config, weights=weights
@@ -661,9 +656,6 @@ class CLIPVisionModel(CLIPPreTrainedModel):
         >>> last_hidden_state = outputs.last_hidden_state
         >>> pooled_output = outputs.pooler_output  # pooled CLS states
         ```"""
-        return_dict = (
-            return_dict if return_dict is not None else self.config.use_return_dict
-        )
 
         return self.vision_model(
             pixel_values=pixel_values,
@@ -799,14 +791,12 @@ class CLIPModel(nn.Module):
         # Use CLIP model's config for some fields (if specified) instead of those of vision & text components.
         vision_outputs = self.vision_model(
             pixel_values=pixel_values,
-            return_dict=return_dict,
         )
 
         text_outputs = self.text_model(
             input_ids=input_ids,
             attention_mask=attention_mask,
             position_ids=position_ids,
-            return_dict=return_dict,
         )
 
         image_embeds = vision_outputs[1]
diff --git a/server/text_generation_server/models/custom_modeling/flash_cohere_modeling.py b/server/text_generation_server/models/custom_modeling/flash_cohere_modeling.py
index bd8b8016..30656038 100644
--- a/server/text_generation_server/models/custom_modeling/flash_cohere_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/flash_cohere_modeling.py
@@ -25,7 +25,12 @@ from torch import nn
 from transformers.activations import ACT2FN
 from typing import Optional, List, Tuple
 
-from text_generation_server.utils import paged_attention, flash_attn
+from text_generation_server.layers.attention import (
+    paged_attention,
+    attention,
+    reshape_and_cache,
+    Seqlen,
+)
 from text_generation_server.utils.import_utils import SYSTEM
 from text_generation_server.layers import (
     TensorParallelRowLinear,
@@ -34,12 +39,14 @@ from text_generation_server.layers import (
     SpeculativeHead,
     get_linear,
 )
+from text_generation_server.layers.attention import PREFILL_IN_KV_CACHE
 from text_generation_server.layers.layernorm import (
     FastLayerNorm,
 )
 from text_generation_server.layers.rotary import (
     PositionRotaryEmbedding,
 )
+from text_generation_server.utils.weights import UnquantizedWeight
 
 if SYSTEM == "cuda":
     import dropout_layer_norm
@@ -78,6 +85,12 @@ class CohereRotary(PositionRotaryEmbedding):
 
             # Inplace operation, updating query and key.
             ops.rotary_embedding(query, key, head_size, cos, sin, False)
+        elif SYSTEM == "ipex":
+            import intel_extension_for_pytorch as ipex
+
+            ipex.llm.functional.rotary_embedding(
+                query, key, sin, cos, query.size(-1), False
+            )
         else:
             raise ValueError(
                 "Your system seem to be not supported. Please check your install or open an issue at https://github.com/huggingface/text-generation-inference/issues with a clear reproduction."
@@ -94,7 +107,7 @@ class CohereLayerNorm(nn.Module):
         self.eps = eps
 
     def forward(self, hidden_states):
-        if hidden_states.shape[-1] > 8192 or SYSTEM == "rocm":
+        if hidden_states.shape[-1] > 8192 or SYSTEM != "cuda":
             hidden_states = hidden_states.reshape(
                 -1, self.weight.shape[0], self.weight.shape[1]
             )
@@ -158,20 +171,19 @@ def _load_gqa(config, prefix: str, weights):
 
     weight = weights.get_multi_weights_col(
         prefixes=[f"{prefix}.q_proj", f"{prefix}.k_proj", f"{prefix}.v_proj"],
-        quantize=config.quantize,
         dim=0,
     )
 
-    if config.quantize not in ["gptq", "awq"]:
-        weight = weight.to(dtype=weights.dtype).to(device=weights.device)
+    if isinstance(weight, UnquantizedWeight):
+        weight.weight = weight.weight.to(dtype=weights.dtype).to(device=weights.device)
 
         head_size = config.hidden_size // config.num_attention_heads
         num_heads = config.num_attention_heads // weights.process_group.size()
         num_key_value_heads = config.num_key_value_heads // weights.process_group.size()
-        assert list(weight.shape) == [
+        assert list(weight.weight.shape) == [
             (num_heads + 2 * num_key_value_heads) * head_size,
             config.hidden_size,
-        ], f"{list(weight.shape)} != {[(num_heads + 2 * config.num_key_value_heads) * head_size, config.hidden_size]}"
+        ], f"{list(weight.weight.shape)} != {[(num_heads + 2 * config.num_key_value_heads) * head_size, config.hidden_size]}"
 
     if config.attention_bias:
         w = [
@@ -182,9 +194,7 @@ def _load_gqa(config, prefix: str, weights):
     else:
         bias = None
 
-    return TensorParallelColumnLinear(
-        get_linear(weight, bias=bias, quantize=config.quantize)
-    )
+    return TensorParallelColumnLinear(get_linear(weight, bias=bias))
 
 
 class FlashCohereAttention(torch.nn.Module):
@@ -256,7 +266,7 @@ class FlashCohereAttention(torch.nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
     ):
         qkv = self.query_key_value(hidden_states)
@@ -281,34 +291,29 @@ class FlashCohereAttention(torch.nn.Module):
 
         self.rotary_emb(query, key, cos, sin)
 
-        paged_attention.reshape_and_cache(key, value, kv_cache[0], kv_cache[1], slots)
-
-        # output tensor
-        attn_output = torch.empty_like(query)
+        reshape_and_cache(key, value, kv_cache[0], kv_cache[1], slots)
 
         # Prefill
         if cu_seqlen_prefill is not None:
             # flash attention
-            flash_attn.attention(
+            attn_output = attention(
                 query,
-                key,
-                value,
-                attn_output,
-                cu_seqlen_prefill,
-                max_s,
+                kv_cache[0] if PREFILL_IN_KV_CACHE else key,
+                kv_cache[1] if PREFILL_IN_KV_CACHE else value,
+                seqlen,
+                block_tables,
                 self.softmax_scale,
             )
         # Decode
         else:
-            paged_attention.attention(
-                attn_output,
+            attn_output = paged_attention(
                 query,
                 kv_cache[0],
                 kv_cache[1],
                 self.kv_head_mapping,
                 self.softmax_scale,
                 block_tables,
-                input_lengths,
+                seqlen,
                 max_s,
             )
 
@@ -358,9 +363,9 @@ class CohereMLP(nn.Module):
 
 
 class FlashCohereLayer(nn.Module):
-    def __init__(self, layer_id, config, weights):
+    def __init__(self, prefix: str, layer_id, config, weights):
         super().__init__()
-        prefix = f"model.layers.{layer_id}"
+        prefix = f"{prefix}.layers.{layer_id}"
         self.self_attn = FlashCohereAttention(
             prefix=f"{prefix}.self_attn", config=config, weights=weights
         )
@@ -383,7 +388,7 @@ class FlashCohereLayer(nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
     ):
         normed_hidden_states, res = self.input_layernorm(hidden_states, residual)
@@ -397,7 +402,7 @@ class FlashCohereLayer(nn.Module):
             kv_cache,
             block_tables,
             slots,
-            input_lengths,
+            seqlen,
             max_s,
         )
 
@@ -411,18 +416,19 @@ class FlashCohereLayer(nn.Module):
 
 
 class FlashCohereModel(torch.nn.Module):
-    def __init__(self, config, weights):
+    def __init__(self, prefix: str, config, weights):
         super().__init__()
 
         process_group = weights.process_group
         self.tp_rank = process_group.rank()
         self.tp_world_size = process_group.size()
         self.embed_tokens = TensorParallelEmbedding(
-            prefix="model.embed_tokens", weights=weights
+            prefix=f"{prefix}.embed_tokens", weights=weights
         )
         self.layers = nn.ModuleList(
             [
                 FlashCohereLayer(
+                    prefix,
                     layer_id,
                     config,
                     weights,
@@ -431,7 +437,7 @@ class FlashCohereModel(torch.nn.Module):
             ]
         )
         self.norm = FastLayerNorm.load_no_bias(
-            prefix="model.norm", weights=weights, eps=config.layer_norm_eps
+            prefix=f"{prefix}.norm", weights=weights, eps=config.layer_norm_eps
         )
 
         self.gradient_checkpointing = False
@@ -448,7 +454,7 @@ class FlashCohereModel(torch.nn.Module):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: torch.Tensor,
         max_s: int,
     ) -> torch.Tensor:
         hidden_states = self.embed_tokens(input_ids)
@@ -460,6 +466,7 @@ class FlashCohereModel(torch.nn.Module):
         )
 
         residual = None
+
         for i, layer in enumerate(self.layers):
             hidden_states, residual = layer(
                 hidden_states,
@@ -470,7 +477,7 @@ class FlashCohereModel(torch.nn.Module):
                 kv_cache[i],
                 block_tables,
                 slots,
-                input_lengths,
+                seqlen,
                 max_s,
             )
 
@@ -480,10 +487,15 @@ class FlashCohereModel(torch.nn.Module):
 
 
 class FlashCohereForCausalLM(torch.nn.Module):
-    def __init__(self, config, weights):
+    def __init__(self, prefix: str, config, weights):
         super().__init__()
 
-        self.model = FlashCohereModel(config, weights)
+        if not prefix:
+            prefix = "model"
+        else:
+            prefix = f"{prefix}.model"
+
+        self.model = FlashCohereModel(prefix, config, weights)
         try:
             self.lm_head = SpeculativeHead.load(
                 config,
@@ -493,7 +505,7 @@ class FlashCohereForCausalLM(torch.nn.Module):
         except RuntimeError:
             self.lm_head = SpeculativeHead.load(
                 config,
-                prefix="model.embed_tokens",
+                prefix=f"{prefix}.embed_tokens",
                 weights=weights,
             )
         self.logit_scale = config.logit_scale
@@ -506,9 +518,11 @@ class FlashCohereForCausalLM(torch.nn.Module):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
+        prefill_cache_indices: Optional[torch.Tensor],
         lm_head_indices: Optional[torch.Tensor] = None,
+        adapter_data: Optional[torch.Tensor] = None,
     ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
         hidden_states = self.model(
             input_ids,
@@ -517,7 +531,7 @@ class FlashCohereForCausalLM(torch.nn.Module):
             kv_cache,
             block_tables,
             slots,
-            input_lengths,
+            seqlen,
             max_s,
         )
         if lm_head_indices is not None:
diff --git a/server/text_generation_server/models/custom_modeling/flash_dbrx_modeling.py b/server/text_generation_server/models/custom_modeling/flash_dbrx_modeling.py
index 9d652b67..1137a453 100644
--- a/server/text_generation_server/models/custom_modeling/flash_dbrx_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/flash_dbrx_modeling.py
@@ -20,13 +20,18 @@ from torch import nn
 from transformers.activations import ACT2FN
 from transformers.configuration_utils import PretrainedConfig
 from typing import Optional, List, Tuple, Any
-from loguru import logger
 from text_generation_server.utils.import_utils import SYSTEM
 
-if SYSTEM != "xpu":
+if SYSTEM != "ipex":
     from vllm.model_executor.layers.fused_moe import fused_moe
 
-from text_generation_server.utils import paged_attention, flash_attn
+from text_generation_server.layers.attention import (
+    paged_attention,
+    attention,
+    reshape_and_cache,
+    Seqlen,
+    PREFILL_IN_KV_CACHE,
+)
 from text_generation_server.layers import (
     FastLinear,
     TensorParallelRowLinear,
@@ -41,7 +46,6 @@ from text_generation_server.layers.rotary import (
 from text_generation_server.layers.layernorm import (
     FastLayerNorm,
 )
-from text_generation_server.utils.log import log_once
 
 
 class DbrxAttentionConfig(PretrainedConfig):
@@ -102,6 +106,12 @@ class DbrxFFNConfig(PretrainedConfig):
 
 
 class DbrxConfig(PretrainedConfig):
+    attribute_map = {
+        "hidden_size": "d_model",
+        "num_attention_heads": "n_heads",
+        "num_hidden_layers": "n_layers",
+    }
+
     def __init__(
         self,
         d_model: int = 2048,
@@ -154,120 +164,25 @@ class DbrxConfig(PretrainedConfig):
             **kwargs,
         )
 
+    @property
+    def num_key_value_heads(self):
+        # We can't use the attribute map, since this the number of KV
+        # heads is not top-level.
+        return self.attn_config.kv_n_heads
+
 
 def promote_scalar(x: torch.Tensor) -> torch.Tensor:
     return x.view(1) if len(x.size()) == 0 else x
 
 
 def load_attention(config, prefix, weights):
-    if config.n_heads != config.attn_config.kv_n_heads:
-        return _load_gqa(config, prefix, weights)
-    else:
-        return TensorParallelColumnLinear.load_qkv(
-            config,
-            prefix=f"{prefix}.Wqkv",
-            weights=weights,
-            bias=False,
-        )
-
-
-def _load_gqa(config, prefix: str, weights):
-    assert config.d_model % config.n_heads == 0
-    assert config.n_heads % weights.process_group.size() == 0
-
-    head_dim = config.d_model // config.n_heads
-    world_size = weights.process_group.size()
-    rank = weights.process_group.rank()
-
-    q_block_size = config.d_model // world_size
-    q_start = rank * q_block_size
-    q_stop = (rank + 1) * q_block_size
-
-    kv_block_size = (config.attn_config.kv_n_heads * head_dim) // world_size
-    k_offset = config.d_model
-    k_start = k_offset + rank * kv_block_size
-    k_stop = k_offset + (rank + 1) * kv_block_size
-
-    v_offset = config.d_model + config.attn_config.kv_n_heads * head_dim
-    v_start = v_offset + rank * kv_block_size
-    v_stop = v_offset + (rank + 1) * kv_block_size
-
-    if config.quantize in ["gptq", "awq"]:
-        try:
-            qweight_slice = weights._get_slice(f"{prefix}.qweight")
-            q_qweight = qweight_slice[:, q_start:q_stop]
-            k_qweight = qweight_slice[:, k_start:k_stop]
-            v_qweight = qweight_slice[:, v_start:v_stop]
-
-            qweight = torch.cat([q_qweight, k_qweight, v_qweight], dim=1)
-        except RuntimeError:
-            raise RuntimeError(
-                f"Cannot load `{config.quantize}` weight, make sure the model is already quantized"
-            )
-
-        qzeros_slice = weights._get_slice(f"{prefix}.qzeros")
-        q_qzeros = qzeros_slice[:, q_start:q_stop]
-        k_qzeros = qzeros_slice[:, k_start:k_stop]
-        v_qzeros = qzeros_slice[:, v_start:v_stop]
-
-        qzeros = torch.cat([q_qzeros, k_qzeros, v_qzeros], dim=1)
-
-        scales_slice = weights._get_slice(f"{prefix}.scales")
-        q_scales = scales_slice[:, q_start:q_stop]
-        k_scales = scales_slice[:, k_start:k_stop]
-        v_scales = scales_slice[:, v_start:v_stop]
-
-        scales = torch.cat([q_scales, k_scales, v_scales], dim=1)
-
-        bits, groupsize, desc_act, quant_method = weights._get_gptq_params()
-
-        from text_generation_server.layers import HAS_EXLLAMA
-
-        use_exllama = (
-            bits == 4 and HAS_EXLLAMA and config.quantize == "gptq" and not desc_act
-        )
-
-        if config.quantize == "gptq" and quant_method == "gptq":
-            g_idx_slice = weights._get_slice(f"{prefix}.g_idx")
-            q_g_idx = g_idx_slice[:, q_start:q_stop]
-            k_g_idx = g_idx_slice[:, k_start:k_stop]
-            v_g_idx = g_idx_slice[:, v_start:v_stop]
-
-            w = [q_g_idx, k_g_idx, v_g_idx]
-            for w2 in w[1:]:
-                torch.testing.assert_close(w2, w[0])
-            g_idx = w[0]
-        elif config.quantize == "gptq" and quant_method == "awq":
-            log_once(
-                logger.info, "Converting AWQ model to Exllama/GPTQ packing format."
-            )
-            from text_generation_server.layers.awq.conveersion_utils import (
-                fast_awq_to_gptq,
-            )
-
-            qweight, qzeros = fast_awq_to_gptq(qweight, qzeros)
-            if use_exllama:
-                g_idx = None
-            else:
-                g_idx = (
-                    torch.arange(qweight.shape[0] * (32 // bits), device=qweight.device)
-                    // groupsize
-                ).to(dtype=torch.int32)
-        else:
-            g_idx = None
-
-        weight = (qweight, qzeros, scales, g_idx, bits, groupsize, use_exllama)
-    else:
-        qkv_slice = weights._get_slice(f"{prefix}.Wqkv.weight")
-        q = qkv_slice[q_start:q_stop]
-        k = qkv_slice[k_start:k_stop]
-        v = qkv_slice[v_start:v_stop]
-
-        weight = torch.cat([q, k, v], dim=0)
-        weight = weight.to(dtype=weights.dtype).to(device=weights.device)
-
-    return TensorParallelColumnLinear(
-        get_linear(weight, bias=None, quantize=config.quantize)
+    return TensorParallelColumnLinear.load_qkv(
+        config,
+        prefix=f"{prefix}.Wqkv",
+        weights=weights,
+        bias=False,
+        num_heads=config.n_heads,
+        num_key_value_heads=config.attn_config.kv_n_heads,
     )
 
 
@@ -333,10 +248,10 @@ def _load_experts_quantized(config, prefix, weights, cls):
 
         if cls == TensorParallelRowLinear:
             expert_slice = expert_slice.t().contiguous()
-            linear = get_linear(expert_slice, None, config.quantize)
+            linear = get_linear(expert_slice, None)
             experts.append(cls(linear, weights.process_group))
         else:
-            linear = get_linear(expert_slice, None, config.quantize)
+            linear = get_linear(expert_slice, None)
             experts.append(cls(linear))
 
     return experts
@@ -396,7 +311,7 @@ class DbrxAttention(torch.nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
     ):
         qkv = self.query_key_value(hidden_states)
@@ -415,36 +330,29 @@ class DbrxAttention(torch.nn.Module):
 
         self.rotary_emb(query, torch.select(kv, dim=1, index=0), cos, sin)
 
-        paged_attention.reshape_and_cache(
-            kv[:, 0], kv[:, 1], kv_cache[0], kv_cache[1], slots
-        )
-
-        # output tensor
-        attn_output = torch.empty_like(query)
+        reshape_and_cache(kv[:, 0], kv[:, 1], kv_cache[0], kv_cache[1], slots)
 
         # Prefill
         if cu_seqlen_prefill is not None:
             # flash attention
-            flash_attn.attention(
+            attn_output = attention(
                 query,
-                torch.select(kv, dim=1, index=0),
-                torch.select(kv, dim=1, index=1),
-                attn_output,
-                cu_seqlen_prefill,
-                max_s,
+                kv_cache[0] if PREFILL_IN_KV_CACHE else kv[:, 0],
+                kv_cache[1] if PREFILL_IN_KV_CACHE else kv[:, 1],
+                seqlen,
+                block_tables,
                 self.softmax_scale,
             )
         # Decode
         else:
-            paged_attention.attention(
-                attn_output,
+            attn_output = paged_attention(
                 query,
                 kv_cache[0],
                 kv_cache[1],
                 self.kv_head_mapping,
                 self.softmax_scale,
                 block_tables,
-                input_lengths,
+                seqlen,
                 max_s,
             )
 
@@ -481,7 +389,7 @@ class DbrxNormAttentionNorm(nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
     ):
         normed_hidden_states, res = self.norm_1(hidden_states, residual)
@@ -495,7 +403,7 @@ class DbrxNormAttentionNorm(nn.Module):
             kv_cache,
             block_tables,
             slots,
-            input_lengths,
+            seqlen,
             max_s,
         )
 
@@ -693,9 +601,9 @@ class DenseMoE(nn.Module):
 
 
 class DbrxLayer(nn.Module):
-    def __init__(self, layer_id, config, weights):
+    def __init__(self, prefix: str, layer_id, config, weights):
         super().__init__()
-        prefix = f"transformer.blocks.{layer_id}"
+        prefix = f"{prefix}.blocks.{layer_id}"
 
         self.attn = DbrxNormAttentionNorm(
             prefix=f"{prefix}.norm_attn_norm", config=config, weights=weights
@@ -714,7 +622,7 @@ class DbrxLayer(nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
     ):
         # Self Attention
@@ -727,7 +635,7 @@ class DbrxLayer(nn.Module):
             kv_cache,
             block_tables,
             slots,
-            input_lengths,
+            seqlen,
             max_s,
         )
 
@@ -737,16 +645,17 @@ class DbrxLayer(nn.Module):
 
 
 class DbrxModel(torch.nn.Module):
-    def __init__(self, config, weights):
+    def __init__(self, prefix: str, config, weights):
         super().__init__()
 
         self.embed_tokens = TensorParallelEmbedding(
-            prefix="transformer.wte", weights=weights
+            prefix=f"{prefix}.wte", weights=weights
         )
 
         self.layers = nn.ModuleList(
             [
                 DbrxLayer(
+                    prefix,
                     layer_id,
                     config,
                     weights,
@@ -755,7 +664,7 @@ class DbrxModel(torch.nn.Module):
             ]
         )
         self.norm = FastLayerNorm.load_no_bias(
-            prefix="transformer.norm_f", weights=weights, eps=1e-5
+            prefix=f"{prefix}.norm_f", weights=weights, eps=1e-5
         )
 
         self.head_size = self.layers[0].attn.self_attn.head_size
@@ -770,7 +679,7 @@ class DbrxModel(torch.nn.Module):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
     ) -> torch.Tensor:
         hidden_states = self.embed_tokens(input_ids)
@@ -792,7 +701,7 @@ class DbrxModel(torch.nn.Module):
                 kv_cache[i],
                 block_tables,
                 slots,
-                input_lengths,
+                seqlen,
                 max_s,
             )
 
@@ -802,10 +711,15 @@ class DbrxModel(torch.nn.Module):
 
 
 class FlashDbrxForCausalLM(torch.nn.Module):
-    def __init__(self, config, weights):
+    def __init__(self, prefix: str, config, weights):
         super().__init__()
 
-        self.model = DbrxModel(config, weights)
+        if not prefix:
+            prefix = "transformer"
+        else:
+            prefix = f"{prefix}.transformer"
+
+        self.model = DbrxModel(prefix, config, weights)
         self.lm_head = SpeculativeHead.load(
             config,
             prefix="lm_head",
@@ -820,9 +734,11 @@ class FlashDbrxForCausalLM(torch.nn.Module):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
+        prefill_cache_indices: Optional[torch.Tensor],
         lm_head_indices: Optional[torch.Tensor] = None,
+        adapter_data: Optional[torch.Tensor] = None,
     ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
         hidden_states = self.model(
             input_ids,
@@ -831,7 +747,7 @@ class FlashDbrxForCausalLM(torch.nn.Module):
             kv_cache,
             block_tables,
             slots,
-            input_lengths,
+            seqlen,
             max_s,
         )
         if lm_head_indices is not None:
diff --git a/server/text_generation_server/models/custom_modeling/flash_deepseek_v2_modeling.py b/server/text_generation_server/models/custom_modeling/flash_deepseek_v2_modeling.py
new file mode 100644
index 00000000..88c2cf80
--- /dev/null
+++ b/server/text_generation_server/models/custom_modeling/flash_deepseek_v2_modeling.py
@@ -0,0 +1,659 @@
+# coding=utf-8
+# Copyright 2023, 2024 DeepSeek-AI and The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import List, Optional, Tuple, Type
+
+import torch
+import torch.distributed
+from torch import nn
+from transformers.activations import ACT2FN
+from transformers.configuration_utils import PretrainedConfig
+
+from text_generation_server.layers import (
+    FastLinear,
+    SpeculativeHead,
+    TensorParallelColumnLinear,
+    TensorParallelEmbedding,
+    TensorParallelRowLinear,
+    get_linear,
+)
+from text_generation_server.layers.attention import (
+    Seqlen,
+    attention,
+    paged_attention,
+    reshape_and_cache,
+)
+from text_generation_server.layers.attention import PREFILL_IN_KV_CACHE
+from text_generation_server.layers.layernorm import FastRMSNorm
+from text_generation_server.layers.moe import DenseMoELayer, MoELayer, SparseMoELayer
+from text_generation_server.layers.rotary import PositionRotaryEmbedding, get_mscale
+from text_generation_server.utils.import_utils import SYSTEM
+from text_generation_server.utils.weights import Weights
+
+if SYSTEM == "rocm":
+    try:
+        from vllm import _custom_C
+    except Exception as e:
+        raise ImportError(f"Could not load `vllm._custom_C`. Full error: {e}")
+
+
+class DeepseekV2Config(PretrainedConfig):
+    def __init__(
+        self,
+        vocab_size=102400,
+        hidden_size=4096,
+        intermediate_size=11008,
+        moe_intermediate_size=1407,
+        num_hidden_layers=30,
+        num_attention_heads=32,
+        num_key_value_heads=32,
+        n_shared_experts=2,
+        n_routed_experts=160,
+        ep_size=1,
+        routed_scaling_factor=1.0,
+        kv_lora_rank=512,
+        q_lora_rank=1536,
+        qk_rope_head_dim=64,
+        v_head_dim=128,
+        qk_nope_head_dim=128,
+        topk_method="gready",
+        n_group=8,
+        topk_group=3,
+        num_experts_per_tok=6,
+        moe_layer_freq=1,
+        first_k_dense_replace=0,
+        norm_topk_prob=False,
+        scoring_func="softmax",
+        aux_loss_alpha=0.001,
+        seq_aux=True,
+        hidden_act="silu",
+        max_position_embeddings=2048,
+        initializer_range=0.02,
+        rms_norm_eps=1e-6,
+        use_cache=True,
+        pad_token_id=None,
+        bos_token_id=100000,
+        eos_token_id=100001,
+        pretraining_tp=1,
+        tie_word_embeddings=False,
+        rope_theta=10000.0,
+        rope_scaling=None,
+        attention_bias=False,
+        attention_dropout=0.0,
+        **kwargs,
+    ):
+        self.vocab_size = vocab_size
+        self.max_position_embeddings = max_position_embeddings
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.moe_intermediate_size = moe_intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.n_shared_experts = n_shared_experts
+        self.n_routed_experts = n_routed_experts
+        self.ep_size = ep_size
+        self.routed_scaling_factor = routed_scaling_factor
+        self.kv_lora_rank = kv_lora_rank
+        self.q_lora_rank = q_lora_rank
+        self.qk_rope_head_dim = qk_rope_head_dim
+        self.v_head_dim = v_head_dim
+        self.qk_nope_head_dim = qk_nope_head_dim
+        self.topk_method = topk_method
+        self.n_group = n_group
+        self.topk_group = topk_group
+        self.num_experts_per_tok = num_experts_per_tok
+        self.moe_layer_freq = moe_layer_freq
+        self.first_k_dense_replace = first_k_dense_replace
+        self.norm_topk_prob = norm_topk_prob
+        self.scoring_func = scoring_func
+        self.aux_loss_alpha = aux_loss_alpha
+        self.seq_aux = seq_aux
+        # for backward compatibility
+        if num_key_value_heads is None:
+            num_key_value_heads = num_attention_heads
+
+        self.num_key_value_heads = num_key_value_heads
+        self.hidden_act = hidden_act
+        self.initializer_range = initializer_range
+        self.rms_norm_eps = rms_norm_eps
+        self.pretraining_tp = pretraining_tp
+        self.use_cache = use_cache
+        self.rope_theta = rope_theta
+        self.rope_scaling = rope_scaling
+        self.attention_bias = attention_bias
+        self.attention_dropout = attention_dropout
+
+        tie_word_embeddings = kwargs.pop("tie_word_embeddings", False)
+        if tie_word_embeddings:
+            raise ValueError(
+                "tie_word_embeddings is not supported for Deepseek V2 models."
+            )
+
+        if ep_size != 1:
+            raise ValueError(
+                f"Currently only ep_size == 1 is supported for Deepseek V2 models, was {ep_size}"
+            )
+
+        super().__init__(
+            pad_token_id=pad_token_id,
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            tie_word_embeddings=tie_word_embeddings,
+            **kwargs,
+        )
+
+
+class DeepseekV2Attention(torch.nn.Module):
+    def __init__(
+        self,
+        prefix: str,
+        config,
+        weights: Weights,
+    ):
+        super().__init__()
+        self.num_heads = config.num_attention_heads
+        self.hidden_size = config.hidden_size
+        self.kv_lora_rank = config.kv_lora_rank
+        self.q_lora_rank = config.q_lora_rank
+        self.qk_nope_head_dim = config.qk_nope_head_dim
+        self.qk_rope_head_dim = config.qk_rope_head_dim
+        self.head_size = config.qk_nope_head_dim + config.qk_rope_head_dim
+        self.value_head_size = config.v_head_dim
+        self.head_pad_size = max(self.head_size, self.value_head_size)
+
+        self.rotary_emb = PositionRotaryEmbedding.static(
+            config=config,
+            dim=self.qk_rope_head_dim,
+            base=config.rope_theta,
+            device=weights.device,
+        )
+
+        mscale = get_mscale(
+            self.rotary_emb.scaling_factor, self.rotary_emb.mscale_all_dim
+        )
+        self.softmax_scale = self.head_size**-0.5 * mscale * mscale
+
+        if self.num_heads % weights.process_group.size() != 0:
+            raise ValueError(
+                f"`num_heads` must be divisible by `num_shards` (got `num_heads`: {self.num_heads} "
+                f"and `num_shards`: {weights.process_group.size()}"
+            )
+        self.num_heads = self.num_heads // weights.process_group.size()
+        self.num_key_value_heads = (
+            config.num_key_value_heads // weights.process_group.size()
+        )
+
+        if self.q_lora_rank is None:
+            self.q_proj = TensorParallelColumnLinear.load(
+                config,
+                prefix=f"{prefix}.q_proj",
+                weights=weights,
+                bias=config.attention_bias,
+            )
+        else:
+            self.q_a_proj = get_linear(
+                weight=weights.get_weights(f"{prefix}.q_a_proj"),
+                bias=(
+                    weights.get_tensor(f"{prefix}.q_a_proj.bias")
+                    if config.attention_bias
+                    else None
+                ),
+            )
+            self.q_a_layernorm = FastRMSNorm.load(
+                prefix=f"{prefix}.q_a_layernorm",
+                weights=weights,
+                eps=config.rms_norm_eps,
+            )
+            self.q_b_proj = TensorParallelColumnLinear.load(
+                config,
+                prefix=f"{prefix}.q_b_proj",
+                weights=weights,
+                bias=config.attention_bias,
+            )
+
+        self.kv_a_proj_with_mqa = get_linear(
+            weight=weights.get_weights(f"{prefix}.kv_a_proj_with_mqa"),
+            bias=(
+                weights.get_tensor(f"{prefix}.kv_a_proj_with_mqa.bias")
+                if config.attention_bias
+                else None
+            ),
+        )
+
+        self.kv_a_layernorm = FastRMSNorm.load(
+            prefix=f"{prefix}.kv_a_layernorm", weights=weights, eps=config.rms_norm_eps
+        )
+
+        self.kv_b_proj = TensorParallelColumnLinear.load(
+            config,
+            prefix=f"{prefix}.kv_b_proj",
+            weights=weights,
+            bias=config.attention_bias,
+        )
+
+        self.o_proj = TensorParallelRowLinear.load(
+            config,
+            prefix=f"{prefix}.o_proj",
+            weights=weights,
+            bias=False,
+        )
+        self.num_groups = self.num_heads // self.num_key_value_heads
+        self.kv_head_mapping = torch.arange(
+            0, self.num_key_value_heads, dtype=torch.int32, device=weights.device
+        ).repeat_interleave(self.num_groups)
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        cos: torch.Tensor,
+        sin: torch.Tensor,
+        cu_seqlen_prefill: torch.Tensor,
+        kv_cache: Tuple[torch.Tensor, torch.Tensor],
+        block_tables: torch.Tensor,
+        slots: torch.Tensor,
+        seqlen: Seqlen,
+        max_s: int,
+    ):
+        if self.q_lora_rank is None:
+            query = self.q_proj(hidden_states)
+        else:
+            query = self.q_b_proj(self.q_a_layernorm(self.q_a_proj(hidden_states))[0])
+        query = query.view(-1, self.num_heads, self.head_size)
+
+        _, query_pe = torch.split(
+            query, [self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1
+        )
+
+        compressed_kv = self.kv_a_proj_with_mqa(hidden_states)
+        compressed_kv, key_pe = torch.split(
+            compressed_kv, [self.kv_lora_rank, self.qk_rope_head_dim], dim=-1
+        )
+
+        key_pe = key_pe.view(-1, 1, self.qk_rope_head_dim)
+        kv = self.kv_b_proj(self.kv_a_layernorm(compressed_kv.contiguous())[0]).view(
+            -1, self.num_key_value_heads, self.qk_nope_head_dim + self.value_head_size
+        )
+
+        key_nope, value = torch.split(
+            kv, [self.qk_nope_head_dim, self.value_head_size], dim=-1
+        )
+
+        batch_size, heads, head_dim = query_pe.shape
+        query_pe = (
+            query_pe.view(batch_size, heads, head_dim // 2, 2)
+            .transpose(2, 3)
+            .reshape(batch_size, heads, head_dim)
+        )
+        batch_size, heads, head_dim = key_pe.shape
+        key_pe = (
+            key_pe.view(batch_size, heads, head_dim // 2, 2)
+            .transpose(2, 3)
+            .reshape(batch_size, heads, head_dim)
+        )
+        self.rotary_emb(query_pe, key_pe, cos, sin)
+
+        query[..., self.qk_nope_head_dim :] = query_pe
+        key = torch.empty_like(query)
+        key[..., : self.qk_nope_head_dim] = key_nope
+        key[..., self.qk_nope_head_dim :] = key_pe
+
+        # We need to pad the heads because Flash Attention does not support
+        # qk and v with different head sizes.
+        query = torch.nn.functional.pad(
+            query, (0, self.head_pad_size - self.head_size), value=0
+        )
+        key = torch.nn.functional.pad(
+            key, (0, self.head_pad_size - self.head_size), value=0
+        )
+        value = torch.nn.functional.pad(
+            value, (0, self.head_pad_size - self.value_head_size), value=0
+        )
+
+        reshape_and_cache(key, value, kv_cache[0], kv_cache[1], slots)
+
+        # Prefill
+        if cu_seqlen_prefill is not None:
+            # flash attention
+            attn_output = attention(
+                query,
+                kv_cache[0] if PREFILL_IN_KV_CACHE else key,
+                kv_cache[1] if PREFILL_IN_KV_CACHE else value,
+                seqlen,
+                block_tables,
+                self.softmax_scale,
+            )
+        # Decode
+        else:
+            attn_output = paged_attention(
+                query,
+                kv_cache[0],
+                kv_cache[1],
+                self.kv_head_mapping,
+                self.softmax_scale,
+                block_tables,
+                seqlen,
+                max_s,
+            )
+
+        # Remove padding.
+        attn_output = attn_output[..., : self.value_head_size]
+
+        return self.o_proj(
+            attn_output.reshape(-1, self.num_heads * self.value_head_size)
+        )
+
+
+class DeepseekV2MLP(nn.Module):
+    def __init__(self, prefix: str, config, weights, intermediate_size: int):
+        super().__init__()
+        self.hidden_act = config.hidden_act
+        if self.hidden_act != "silu":
+            # Bail out because MoE only supports silu.
+            raise NotImplementedError(
+                "Currently only `silu` is supported as an activation for Deepseek V2."
+            )
+        self.act = ACT2FN[self.hidden_act]
+
+        self.gate_up_proj = TensorParallelColumnLinear.load_multi(
+            config,
+            prefixes=[f"{prefix}.gate_proj", f"{prefix}.up_proj"],
+            weights=weights,
+            dim=0,
+            bias=False,
+        )
+
+        self.down_proj = TensorParallelRowLinear.load(
+            config,
+            prefix=f"{prefix}.down_proj",
+            weights=weights,
+            bias=False,
+        )
+
+        self.intermediate_size = intermediate_size // weights.process_group.size()
+
+        # TODO: This is a hotfix to be removed & properly refactored.
+        self.quantize = config.quantize
+
+    def forward(self, hidden_states: torch.Tensor, reduce: bool = True):
+        if (
+            SYSTEM == "rocm"
+            and self.hidden_act == "silu"
+            and hidden_states.dtype == torch.float16
+            and hidden_states.shape[0] == 1
+            and not self.quantize
+        ):
+            out = torch.empty(
+                hidden_states.shape[0],
+                self.intermediate_size,
+                dtype=hidden_states.dtype,
+                device="cuda",
+            )
+            _custom_C.LLMM_Silu(self.gate_up_proj.linear.weight, hidden_states, out, 8)
+            return self.down_proj(out, reduce=reduce)
+        else:
+            gate_up_states = self.gate_up_proj(hidden_states)
+            gate_up_states = gate_up_states.view(-1, 2, self.intermediate_size)
+            return self.down_proj(
+                self.act(gate_up_states[:, 0]) * gate_up_states[:, 1], reduce=reduce
+            )
+
+
+class DeepseekV2MoE(nn.Module):
+    def __init__(
+        self,
+        prefix,
+        config: DeepseekV2Config,
+        moe_layer_cls: Type[MoELayer],
+        weights,
+    ):
+        super().__init__()
+
+        self.hidden_dim = config.hidden_size
+        self.moe_intermediate_size = (
+            config.moe_intermediate_size // weights.process_group.size()
+        )
+        self.routed_scaling_factor = config.routed_scaling_factor
+
+        # Gating
+        self.gate = FastLinear.load(config, f"{prefix}.gate", weights, bias=False)
+
+        self.moe_layer = moe_layer_cls(
+            prefix=f"{prefix}.experts",
+            n_experts=config.n_routed_experts,
+            n_expert_group=config.n_group,
+            renormalize=config.norm_topk_prob,
+            topk=config.num_experts_per_tok,
+            topk_group=config.topk_group,
+            weights=weights,
+        )
+        assert isinstance(self.moe_layer, MoELayer)
+
+        if config.n_shared_experts is not None:
+            self.shared_experts = DeepseekV2MLP(
+                prefix=f"{prefix}.shared_experts",
+                config=config,
+                weights=weights,
+                intermediate_size=config.moe_intermediate_size
+                * config.n_shared_experts,
+            )
+        else:
+            self.shared_experts = None
+
+        self.process_group = weights.process_group
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        if self.shared_experts is not None:
+            shared_output = self.shared_experts(x, reduce=False)
+        else:
+            shared_output = None
+
+        router_logits = self.gate(x)
+
+        out = self.moe_layer(x, gating_output=router_logits)
+
+        if shared_output is not None:
+            out = out + shared_output
+
+        # Reduce sum
+        if self.process_group.size() > 1:
+            torch.distributed.all_reduce(out, group=self.process_group)
+
+        return out.view(*x.shape)
+
+
+class DeepseekV2Layer(nn.Module):
+    def __init__(self, prefix, layer_id, config, weights):
+        super().__init__()
+        prefix = f"{prefix}.layers.{layer_id}"
+
+        self.self_attn = DeepseekV2Attention(
+            prefix=f"{prefix}.self_attn",
+            config=config,
+            weights=weights,
+        )
+
+        if (
+            config.n_routed_experts is not None
+            and layer_id >= config.first_k_dense_replace
+            and layer_id % config.moe_layer_freq == 0
+        ):
+            moe_layer_cls = (
+                SparseMoELayer
+                if SparseMoELayer.is_supported(weights)
+                else DenseMoELayer
+            )
+            self.mlp = DeepseekV2MoE(f"{prefix}.mlp", config, moe_layer_cls, weights)
+        else:
+            self.mlp = DeepseekV2MLP(
+                prefix=f"{prefix}.mlp",
+                config=config,
+                weights=weights,
+                intermediate_size=config.intermediate_size,
+            )
+
+        self.input_layernorm = FastRMSNorm.load(
+            prefix=f"{prefix}.input_layernorm", weights=weights, eps=config.rms_norm_eps
+        )
+        self.post_attention_layernorm = FastRMSNorm.load(
+            prefix=f"{prefix}.post_attention_layernorm",
+            weights=weights,
+            eps=config.rms_norm_eps,
+        )
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        residual: torch.Tensor,
+        cos: torch.Tensor,
+        sin: torch.Tensor,
+        cu_seqlen_prefill: torch.Tensor,
+        kv_cache,
+        block_tables: torch.Tensor,
+        slots: torch.Tensor,
+        seqlen: Seqlen,
+        max_s: int,
+    ):
+        normed_hidden_states, residual = self.input_layernorm(hidden_states, residual)
+
+        # Self Attention
+        attn_output = self.self_attn(
+            normed_hidden_states,
+            cos,
+            sin,
+            cu_seqlen_prefill,
+            kv_cache,
+            block_tables,
+            slots,
+            seqlen,
+            max_s,
+        )
+
+        # faster post attention rms norm
+        normed_attn_res_output, residual = self.post_attention_layernorm(
+            attn_output, residual
+        )
+
+        output = self.mlp(normed_attn_res_output)
+
+        return output, residual
+
+
+class DeepseekV2Model(torch.nn.Module):
+    def __init__(self, prefix: str, config, weights: Weights):
+        super().__init__()
+
+        self.embed_tokens = TensorParallelEmbedding(
+            prefix=f"{prefix}.embed_tokens", weights=weights
+        )
+
+        self.layers = nn.ModuleList(
+            [
+                DeepseekV2Layer(
+                    prefix,
+                    layer_id,
+                    config,
+                    weights,
+                )
+                for layer_id in range(config.num_hidden_layers)
+            ]
+        )
+        self.norm = FastRMSNorm.load(
+            prefix=f"{prefix}.norm", weights=weights, eps=config.rms_norm_eps
+        )
+
+        self.head_size = self.layers[0].self_attn.head_size
+        self.num_heads = self.layers[0].self_attn.num_heads
+        self.num_key_value_heads = self.layers[0].self_attn.num_key_value_heads
+
+    def forward(
+        self,
+        input_ids: torch.Tensor,
+        position_ids: torch.Tensor,
+        cu_seqlen_prefill: Optional[torch.Tensor],
+        kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
+        block_tables: torch.Tensor,
+        slots: torch.Tensor,
+        seqlen: Seqlen,
+        max_s: int,
+    ) -> torch.Tensor:
+        hidden_states = self.embed_tokens(input_ids)
+
+        # Get rotary cos and sin for this forward
+        # Avoid to index in each layer
+        cos, sin = self.layers[0].self_attn.rotary_emb.get_cos_sin(
+            position_ids, max_s, hidden_states.dtype
+        )
+
+        residual = None
+        for i, layer in enumerate(self.layers):
+            hidden_states, residual = layer(
+                hidden_states,
+                residual,
+                cos,
+                sin,
+                cu_seqlen_prefill,
+                kv_cache[i],
+                block_tables,
+                slots,
+                seqlen,
+                max_s,
+            )
+
+        hidden_states, _ = self.norm(hidden_states, residual)
+
+        return hidden_states
+
+
+class FlashDeepseekV2ForCausalLM(torch.nn.Module):
+    def __init__(self, prefix: str, config, weights: Weights):
+        super().__init__()
+
+        self.model = DeepseekV2Model(
+            "model" if not prefix else f"{prefix}.model", config, weights
+        )
+        self.lm_head = SpeculativeHead.load(
+            config,
+            prefix="lm_head" if not prefix else f"{prefix}.lm_head",
+            weights=weights,
+        )
+
+    def forward(
+        self,
+        input_ids: torch.Tensor,
+        position_ids: torch.Tensor,
+        cu_seqlen_prefill: Optional[torch.Tensor],
+        kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
+        block_tables: torch.Tensor,
+        slots: torch.Tensor,
+        seqlen: Seqlen,
+        max_s: int,
+        prefill_cache_indices: Optional[torch.Tensor],
+        lm_head_indices: Optional[torch.Tensor] = None,
+        adapter_data: Optional[torch.Tensor] = None,
+    ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
+        hidden_states = self.model(
+            input_ids,
+            position_ids,
+            cu_seqlen_prefill,
+            kv_cache,
+            block_tables,
+            slots,
+            seqlen,
+            max_s,
+        )
+        if lm_head_indices is not None:
+            hidden_states = hidden_states[lm_head_indices]
+        logits, speculative_logits = self.lm_head(hidden_states)
+        return logits, speculative_logits
diff --git a/server/text_generation_server/models/custom_modeling/flash_gemma2_modeling.py b/server/text_generation_server/models/custom_modeling/flash_gemma2_modeling.py
new file mode 100644
index 00000000..7a3d60c9
--- /dev/null
+++ b/server/text_generation_server/models/custom_modeling/flash_gemma2_modeling.py
@@ -0,0 +1,560 @@
+# coding=utf-8
+# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
+#
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
+# and OPT implementations in this library. It has been modified from its
+# original forms to accommodate minor architectural differences compared
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import torch
+import torch.distributed
+
+from torch import nn
+from transformers.activations import ACT2FN
+from transformers.configuration_utils import PretrainedConfig
+from typing import Optional, List, Tuple
+from text_generation_server.layers.attention import (
+    paged_attention,
+    attention,
+    reshape_and_cache,
+    Seqlen,
+)
+from text_generation_server.layers import (
+    TensorParallelRowLinear,
+    TensorParallelColumnLinear,
+    TensorParallelEmbedding,
+    SpeculativeHead,
+    get_linear,
+    TensorParallelMultiAdapterLinear,
+    TensorParallelAdapterRowLinear,
+)
+from text_generation_server.layers.attention import PREFILL_IN_KV_CACHE
+from text_generation_server.layers.rotary import PositionRotaryEmbedding
+from text_generation_server.layers.layernorm import (
+    FastRMSNorm,
+)
+from text_generation_server.utils.weights import UnquantizedWeight
+
+
+class Gemma2Config(PretrainedConfig):
+    def __init__(
+        self,
+        vocab_size=256128,
+        hidden_size=3072,
+        intermediate_size=24576,
+        num_hidden_layers=28,
+        num_attention_heads=16,
+        num_key_value_heads=16,
+        head_dim=256,
+        hidden_act="gelu_pytorch_tanh",
+        max_position_embeddings=8192,
+        initializer_range=0.02,
+        rms_norm_eps=1e-6,
+        use_cache=True,
+        pad_token_id=None,
+        bos_token_id=1,
+        eos_token_id=2,
+        tie_word_embeddings=True,
+        rope_theta=10000.0,
+        rope_scaling=None,
+        attention_bias=False,
+        attention_dropout=0.0,
+        **kwargs,
+    ):
+        self.vocab_size = vocab_size
+        self.max_position_embeddings = max_position_embeddings
+        self.hidden_size = hidden_size
+        self.head_dim = head_dim
+        self.intermediate_size = intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+
+        # for backward compatibility
+        if num_key_value_heads is None:
+            num_key_value_heads = num_attention_heads
+
+        self.num_key_value_heads = num_key_value_heads
+        self.hidden_act = hidden_act
+        self.initializer_range = initializer_range
+        self.rms_norm_eps = rms_norm_eps
+        self.use_cache = use_cache
+        self.rope_theta = rope_theta
+        self.rope_scaling = rope_scaling
+        self.attention_bias = attention_bias
+        self.attention_dropout = attention_dropout
+
+        super().__init__(
+            pad_token_id=pad_token_id,
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            tie_word_embeddings=tie_word_embeddings,
+            **kwargs,
+        )
+
+
+class Gemma2FastRMSNorm(FastRMSNorm):
+    @classmethod
+    def load(cls, prefix: str, weights, eps=1e-6):
+        dtype = weights.dtype
+        weights.dtype = torch.float32
+        weight = weights.get_tensor(f"{prefix}.weight") + 1
+        weights.dtype = dtype
+        new = cls(weight, eps)
+        new.dtype = dtype
+        return new
+
+    # perform the multiplication in full precision and downcast after
+    def forward(self, hidden_states, residual=None):
+        if residual is not None:
+            hidden_states += residual
+        residual = hidden_states
+        hidden_states = hidden_states.to(torch.float32)
+        variance = hidden_states.pow(2).mean(-1, keepdim=True)
+        hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
+        hidden_states = hidden_states * self.weight
+        return hidden_states.to(self.dtype), residual
+
+
+def load_attention(config, prefix: str, weights):
+    if config.num_attention_heads != config.num_key_value_heads:
+        return _load_gqa(config, prefix, weights)
+    else:
+        return TensorParallelColumnLinear.load_multi(
+            config,
+            prefixes=[f"{prefix}.q_proj", f"{prefix}.k_proj", f"{prefix}.v_proj"],
+            dim=0,
+            weights=weights,
+            bias=False,
+        )
+
+
+def _load_gqa(config, prefix: str, weights):
+    assert config.num_attention_heads % weights.process_group.size() == 0
+
+    weight = weights.get_multi_weights_col(
+        prefixes=[f"{prefix}.q_proj", f"{prefix}.k_proj", f"{prefix}.v_proj"],
+        dim=0,
+    )
+
+    if isinstance(weight, UnquantizedWeight):
+        weight.weight = weight.weight.to(dtype=weights.dtype).to(device=weights.device)
+
+        head_size = config.head_dim
+        num_heads = config.num_attention_heads // weights.process_group.size()
+        num_key_value_heads = config.num_key_value_heads // weights.process_group.size()
+        assert list(weight.weight.shape) == [
+            (num_heads + 2 * num_key_value_heads) * head_size,
+            config.hidden_size,
+        ], f"{list(weight.weight.shape)} != {[(num_heads + 2 * config.num_key_value_heads) * head_size, config.hidden_size]}"
+
+    return TensorParallelColumnLinear(get_linear(weight, bias=None))
+
+
+class FlashGemma2Attention(torch.nn.Module):
+    def __init__(
+        self, prefix: str, config, weights, layer_id, causal: bool, is_sliding: bool
+    ):
+        super().__init__()
+        self.num_heads = config.num_attention_heads
+        self.head_size = config.head_dim
+        self.causal = causal
+        if is_sliding:
+            self.window_size = config.sliding_window
+        else:
+            self.window_size = -1
+
+        self.rotary_emb = PositionRotaryEmbedding.static(
+            config=config,
+            dim=self.head_size,
+            base=config.rope_theta,
+            device=weights.device,
+        )
+
+        # self.softmax_scale = self.head_size**-0.5
+        self.softmax_scale = config.query_pre_attn_scalar**-0.5
+
+        if self.num_heads % weights.process_group.size() != 0:
+            raise ValueError(
+                f"`num_heads` must be divisible by `num_shards` (got `num_heads`: {self.num_heads} "
+                f"and `num_shards`: {weights.process_group.size()}"
+            )
+        self.num_heads = self.num_heads // weights.process_group.size()
+        self.num_key_value_heads = (
+            config.num_key_value_heads // weights.process_group.size()
+        )
+        self.softcap = config.attn_logit_softcapping
+
+        query_key_value = load_attention(config, prefix, weights)
+        self.query_key_value = TensorParallelMultiAdapterLinear.load(
+            query_key_value,
+            layer_id,
+            ["q_proj", "k_proj", "v_proj"],
+            sizes=[
+                self.head_size * config.num_attention_heads,
+                self.head_size * config.num_key_value_heads,
+                self.head_size * config.num_key_value_heads,
+            ],
+            process_group=weights.process_group,
+        )
+
+        o_proj = TensorParallelRowLinear.load(
+            config,
+            prefix=f"{prefix}.o_proj",
+            weights=weights,
+            bias=False,
+        )
+        self.o_proj = TensorParallelAdapterRowLinear.load(
+            o_proj,
+            layer_id,
+            "o_proj",
+            process_group=weights.process_group,
+        )
+
+        self.num_groups = self.num_heads // self.num_key_value_heads
+        self.kv_head_mapping = torch.arange(
+            0, self.num_key_value_heads, dtype=torch.int32, device=weights.device
+        ).repeat_interleave(self.num_groups)
+
+    def forward(
+        self,
+        hidden_states,
+        cos,
+        sin,
+        cu_seqlen_prefill,
+        kv_cache,
+        block_tables,
+        slots,
+        seqlen,
+        max_s,
+        adapter_data,
+    ):
+        qkv = self.query_key_value(hidden_states, adapter_data)
+        query, kv = qkv.split(
+            [
+                self.head_size * self.num_heads,
+                2 * self.head_size * self.num_key_value_heads,
+            ],
+            dim=1,
+        )
+        query = query.view(-1, self.num_heads, self.head_size)
+        kv = kv.view(-1, 2, self.num_key_value_heads, self.head_size)
+
+        self.rotary_emb(query, torch.select(kv, dim=1, index=0), cos, sin)
+
+        reshape_and_cache(kv[:, 0], kv[:, 1], kv_cache[0], kv_cache[1], slots)
+
+        # Prefill
+        if cu_seqlen_prefill is not None:
+            # flash attention
+            attn_output = attention(
+                query,
+                kv_cache[0] if PREFILL_IN_KV_CACHE else kv[:, 0],
+                kv_cache[1] if PREFILL_IN_KV_CACHE else kv[:, 1],
+                seqlen,
+                block_tables,
+                self.softmax_scale,
+                causal=self.causal,
+                window_size_left=self.window_size,
+                softcap=self.softcap,
+            )
+        # Decode
+        else:
+            attn_output = paged_attention(
+                query,
+                kv_cache[0],
+                kv_cache[1],
+                self.kv_head_mapping,
+                self.softmax_scale,
+                block_tables,
+                seqlen,
+                max_s,
+                softcap=self.softcap,
+            )
+
+        return self.o_proj(
+            attn_output.view(-1, self.num_heads * self.head_size), adapter_data
+        )
+
+
+class Gemma2MLP(nn.Module):
+    def __init__(self, prefix, config, weights, layer_id):
+        super().__init__()
+        act = config.hidden_activation
+        self.act = (
+            ACT2FN[act]
+            if "gelu" not in act
+            else lambda x: torch.nn.functional.gelu(
+                x,
+                approximate=(
+                    "tanh" if act in ["gelu_fast", "gelu_pytorch_tanh"] else "none"
+                ),
+            )
+        )
+        # Fuse gate and up proj
+        gate_up_proj = TensorParallelColumnLinear.load_multi(
+            config,
+            prefixes=[f"{prefix}.gate_proj", f"{prefix}.up_proj"],
+            weights=weights,
+            dim=0,
+            bias=False,
+        )
+        self.gate_up_proj = TensorParallelMultiAdapterLinear.load(
+            gate_up_proj,
+            layer_id,
+            ["gate_proj", "up_proj"],
+            sizes=[
+                config.intermediate_size,
+                config.intermediate_size,
+            ],
+            process_group=weights.process_group,
+        )
+
+        down_proj = TensorParallelRowLinear.load(
+            config,
+            prefix=f"{prefix}.down_proj",
+            weights=weights,
+            bias=False,
+        )
+        self.down_proj = TensorParallelAdapterRowLinear.load(
+            down_proj,
+            layer_id,
+            "down_proj",
+            process_group=weights.process_group,
+        )
+
+        self.intermediate_size = (
+            config.intermediate_size // weights.process_group.size()
+        )
+
+    def forward(self, hidden_states, adapter_data):
+        gate_up_states = self.gate_up_proj(hidden_states, adapter_data)
+        gate_up_states = gate_up_states.view(-1, 2, self.intermediate_size)
+        return self.down_proj(
+            self.act(gate_up_states[:, 0]) * gate_up_states[:, 1], adapter_data
+        )
+
+
+class FlashGemma2Layer(nn.Module):
+    def __init__(
+        self, prefix: str, config, weights, layer_id, causal: bool, is_sliding: bool
+    ):
+        super().__init__()
+        self.self_attn = FlashGemma2Attention(
+            prefix=f"{prefix}.self_attn",
+            config=config,
+            weights=weights,
+            layer_id=layer_id,
+            causal=causal,
+            is_sliding=is_sliding,
+        )
+        self.mlp = Gemma2MLP(
+            prefix=f"{prefix}.mlp", config=config, weights=weights, layer_id=layer_id
+        )
+
+        self.input_layernorm = Gemma2FastRMSNorm.load(
+            prefix=f"{prefix}.input_layernorm", weights=weights, eps=config.rms_norm_eps
+        )
+        self.post_attention_layernorm = Gemma2FastRMSNorm.load(
+            prefix=f"{prefix}.post_attention_layernorm",
+            weights=weights,
+            eps=config.rms_norm_eps,
+        )
+        self.pre_feedforward_layernorm = Gemma2FastRMSNorm.load(
+            prefix=f"{prefix}.pre_feedforward_layernorm",
+            weights=weights,
+            eps=config.rms_norm_eps,
+        )
+        self.post_feedforward_layernorm = Gemma2FastRMSNorm.load(
+            prefix=f"{prefix}.post_feedforward_layernorm",
+            weights=weights,
+            eps=config.rms_norm_eps,
+        )
+
+    def forward(
+        self,
+        hidden_states,
+        residual,
+        cos,
+        sin,
+        cu_seqlen_prefill,
+        kv_cache,
+        block_tables,
+        slots,
+        seqlen,
+        max_s,
+        adapter_data,
+    ):
+        normed_hidden_states, res = self.input_layernorm(hidden_states, residual)
+
+        # Self Attention
+        attn_output = self.self_attn(
+            normed_hidden_states,
+            cos,
+            sin,
+            cu_seqlen_prefill,
+            kv_cache,
+            block_tables,
+            slots,
+            seqlen,
+            max_s,
+            adapter_data,
+        )
+
+        # faster post attention rms norm
+        normed_attn_res_output, _ = self.post_attention_layernorm(attn_output)
+        normed_attn_res_output = normed_attn_res_output + res
+        res = normed_attn_res_output
+
+        pre_normed, _ = self.pre_feedforward_layernorm(normed_attn_res_output)
+        mlp_output = self.mlp(pre_normed, adapter_data)
+        post_hidden_states, _ = self.post_feedforward_layernorm(mlp_output)
+
+        return post_hidden_states, normed_attn_res_output
+
+
+class FlashGemma2Model(torch.nn.Module):
+    def __init__(self, prefix: str, config, weights, causal: bool):
+        super().__init__()
+
+        process_group = weights.process_group
+        self.tp_rank = process_group.rank()
+        self.tp_world_size = process_group.size()
+        self.layers = nn.ModuleList(
+            [
+                FlashGemma2Layer(
+                    prefix=f"{prefix}.layers.{layer_id}",
+                    config=config,
+                    weights=weights,
+                    layer_id=layer_id,
+                    causal=causal,
+                    is_sliding=layer_id % 2 == 0,
+                )
+                for layer_id in range(config.num_hidden_layers)
+            ]
+        )
+        self.norm = Gemma2FastRMSNorm.load(
+            prefix=f"{prefix}.norm", weights=weights, eps=config.rms_norm_eps
+        )
+
+        self.head_size = self.layers[0].self_attn.head_size
+        self.num_heads = self.layers[0].self_attn.num_heads
+        self.num_key_value_heads = self.layers[0].self_attn.num_key_value_heads
+
+    def forward(
+        self,
+        inputs_embeds: torch.Tensor,
+        position_ids: torch.Tensor,
+        cu_seqlen_prefill: Optional[torch.Tensor],
+        kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
+        block_tables: torch.Tensor,
+        slots: torch.Tensor,
+        seqlen: Seqlen,
+        max_s: int,
+        adapter_data: Optional[torch.Tensor] = None,
+    ) -> torch.Tensor:
+        hidden_states = inputs_embeds
+
+        # Get rotary cos and sin for this forward
+        # Avoid to index in each layer
+        cos, sin = self.layers[0].self_attn.rotary_emb.get_cos_sin(
+            position_ids, max_s, hidden_states.dtype
+        )
+
+        residual = None
+        for i, layer in enumerate(self.layers):
+            hidden_states, residual = layer(
+                hidden_states,
+                residual,
+                cos,
+                sin,
+                cu_seqlen_prefill,
+                kv_cache[i],
+                block_tables,
+                slots,
+                seqlen,
+                max_s,
+                adapter_data,
+            )
+
+        hidden_states, _ = self.norm(hidden_states, residual)
+
+        return hidden_states
+
+
+class FlashGemma2ForCausalLM(torch.nn.Module):
+    def __init__(self, prefix: str, config, weights, *, causal: bool = True):
+        super().__init__()
+
+        embed_norm = config.hidden_size**0.5
+        if not prefix:
+            prefix = "model"
+        else:
+            prefix = f"{prefix}.model"
+
+        self.embed_tokens = TensorParallelEmbedding(
+            prefix=f"{prefix}.embed_tokens", weights=weights
+        )
+        self.embed_tokens.weight *= embed_norm
+
+        self.model = FlashGemma2Model(
+            prefix=prefix, config=config, weights=weights, causal=causal
+        )
+        self.lm_head = SpeculativeHead.load(
+            prefix=(
+                f"{prefix}.embed_tokens"
+                if config.tie_word_embeddings
+                else f"{prefix}.lm_head"
+            ),
+            config=config,
+            weights=weights,
+        )
+        self.softcap = config.final_logit_softcapping
+        assert isinstance(self.softcap, float)
+
+    def forward(
+        self,
+        input_ids: torch.Tensor,
+        position_ids: torch.Tensor,
+        cu_seqlen_prefill: Optional[torch.Tensor],
+        kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
+        block_tables: torch.Tensor,
+        slots: torch.Tensor,
+        seqlen: Seqlen,
+        max_s: int,
+        prefill_cache_indices: Optional[torch.Tensor],
+        lm_head_indices: Optional[torch.Tensor] = None,
+        adapter_data: Optional[torch.Tensor] = None,
+    ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
+        input_embeds = self.embed_tokens(input_ids)
+        hidden_states = self.model(
+            input_embeds,
+            position_ids,
+            cu_seqlen_prefill,
+            kv_cache,
+            block_tables,
+            slots,
+            seqlen,
+            max_s,
+            adapter_data,
+        )
+        if lm_head_indices is not None:
+            hidden_states = hidden_states[lm_head_indices]
+        logits, speculative_logits = self.lm_head(hidden_states)
+
+        logits /= self.softcap
+        logits = torch.tanh(logits)
+        logits *= self.softcap
+
+        return logits, speculative_logits
diff --git a/server/text_generation_server/models/custom_modeling/flash_gemma_modeling.py b/server/text_generation_server/models/custom_modeling/flash_gemma_modeling.py
index ac6fd0e6..4c1be6f6 100644
--- a/server/text_generation_server/models/custom_modeling/flash_gemma_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/flash_gemma_modeling.py
@@ -25,8 +25,13 @@ from torch import nn
 from transformers.activations import ACT2FN
 from transformers.configuration_utils import PretrainedConfig
 from typing import Optional, List, Tuple
-
-from text_generation_server.utils import paged_attention, flash_attn
+from text_generation_server.layers.attention import (
+    paged_attention,
+    attention,
+    reshape_and_cache,
+    Seqlen,
+    PREFILL_IN_KV_CACHE,
+)
 from text_generation_server.layers import (
     TensorParallelRowLinear,
     TensorParallelColumnLinear,
@@ -38,6 +43,7 @@ from text_generation_server.layers.rotary import PositionRotaryEmbedding
 from text_generation_server.layers.layernorm import (
     FastRMSNorm,
 )
+from text_generation_server.utils.weights import UnquantizedWeight
 
 
 class GemmaConfig(PretrainedConfig):
@@ -98,7 +104,7 @@ class GemmaConfig(PretrainedConfig):
 
 class GemmaFastRMSNorm(FastRMSNorm):
     @classmethod
-    def load(cls, prefix, weights, eps=1e-6):
+    def load(cls, prefix: str, weights, eps=1e-6):
         dtype = weights.dtype
         weights.dtype = torch.float32
         weight = weights.get_tensor(f"{prefix}.weight") + 1
@@ -119,7 +125,7 @@ class GemmaFastRMSNorm(FastRMSNorm):
         return hidden_states.to(self.dtype), residual
 
 
-def load_attention(config, prefix, weights):
+def load_attention(config, prefix: str, weights):
     if config.num_attention_heads != config.num_key_value_heads:
         return _load_gqa(config, prefix, weights)
     else:
@@ -137,24 +143,21 @@ def _load_gqa(config, prefix: str, weights):
 
     weight = weights.get_multi_weights_col(
         prefixes=[f"{prefix}.q_proj", f"{prefix}.k_proj", f"{prefix}.v_proj"],
-        quantize=config.quantize,
         dim=0,
     )
 
-    if config.quantize not in ["gptq", "awq"]:
-        weight = weight.to(dtype=weights.dtype).to(device=weights.device)
+    if isinstance(weight, UnquantizedWeight):
+        weight.weight = weight.weight.to(dtype=weights.dtype).to(device=weights.device)
 
         head_size = config.head_dim
         num_heads = config.num_attention_heads // weights.process_group.size()
         num_key_value_heads = config.num_key_value_heads // weights.process_group.size()
-        assert list(weight.shape) == [
+        assert list(weight.weight.shape) == [
             (num_heads + 2 * num_key_value_heads) * head_size,
             config.hidden_size,
-        ], f"{list(weight.shape)} != {[(num_heads + 2 * config.num_key_value_heads) * head_size, config.hidden_size]}"
+        ], f"{list(weight.weight.shape)} != {[(num_heads + 2 * config.num_key_value_heads) * head_size, config.hidden_size]}"
 
-    return TensorParallelColumnLinear(
-        get_linear(weight, bias=None, quantize=config.quantize)
-    )
+    return TensorParallelColumnLinear(get_linear(weight, bias=None))
 
 
 class FlashGemmaAttention(torch.nn.Module):
@@ -205,7 +208,7 @@ class FlashGemmaAttention(torch.nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
     ):
         qkv = self.query_key_value(hidden_states)
@@ -221,37 +224,30 @@ class FlashGemmaAttention(torch.nn.Module):
 
         self.rotary_emb(query, torch.select(kv, dim=1, index=0), cos, sin)
 
-        paged_attention.reshape_and_cache(
-            kv[:, 0], kv[:, 1], kv_cache[0], kv_cache[1], slots
-        )
-
-        # output tensor
-        attn_output = torch.empty_like(query)
+        reshape_and_cache(kv[:, 0], kv[:, 1], kv_cache[0], kv_cache[1], slots)
 
         # Prefill
         if cu_seqlen_prefill is not None:
             # flash attention
-            flash_attn.attention(
+            attn_output = attention(
                 query,
-                torch.select(kv, dim=1, index=0),
-                torch.select(kv, dim=1, index=1),
-                attn_output,
-                cu_seqlen_prefill,
-                max_s,
+                kv_cache[0] if PREFILL_IN_KV_CACHE else kv[:, 0],
+                kv_cache[1] if PREFILL_IN_KV_CACHE else kv[:, 1],
+                seqlen,
+                block_tables,
                 self.softmax_scale,
                 causal=self.causal,
             )
         # Decode
         else:
-            paged_attention.attention(
-                attn_output,
+            attn_output = paged_attention(
                 query,
                 kv_cache[0],
                 kv_cache[1],
                 self.kv_head_mapping,
                 self.softmax_scale,
                 block_tables,
-                input_lengths,
+                seqlen,
                 max_s,
             )
 
@@ -259,7 +255,7 @@ class FlashGemmaAttention(torch.nn.Module):
 
 
 class GemmaMLP(nn.Module):
-    def __init__(self, prefix, config, weights):
+    def __init__(self, prefix: str, config, weights):
         super().__init__()
         act = config.hidden_act
         self.act = (
@@ -297,7 +293,7 @@ class GemmaMLP(nn.Module):
 
 
 class FlashGemmaLayer(nn.Module):
-    def __init__(self, prefix, config, weights, causal: bool):
+    def __init__(self, prefix: str, config, weights, causal: bool):
         super().__init__()
         self.self_attn = FlashGemmaAttention(
             prefix=f"{prefix}.self_attn", config=config, weights=weights, causal=causal
@@ -323,7 +319,7 @@ class FlashGemmaLayer(nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
     ):
         normed_hidden_states, res = self.input_layernorm(hidden_states, residual)
@@ -337,7 +333,7 @@ class FlashGemmaLayer(nn.Module):
             kv_cache,
             block_tables,
             slots,
-            input_lengths,
+            seqlen,
             max_s,
         )
 
@@ -352,7 +348,7 @@ class FlashGemmaLayer(nn.Module):
 
 
 class FlashGemmaModel(torch.nn.Module):
-    def __init__(self, prefix, config, weights, causal: bool):
+    def __init__(self, prefix: str, config, weights, causal: bool):
         super().__init__()
 
         process_group = weights.process_group
@@ -373,8 +369,6 @@ class FlashGemmaModel(torch.nn.Module):
             prefix=f"{prefix}.norm", weights=weights, eps=config.rms_norm_eps
         )
 
-        self.gradient_checkpointing = False
-
         self.head_size = self.layers[0].self_attn.head_size
         self.num_heads = self.layers[0].self_attn.num_heads
         self.num_key_value_heads = self.layers[0].self_attn.num_key_value_heads
@@ -387,7 +381,7 @@ class FlashGemmaModel(torch.nn.Module):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
     ) -> torch.Tensor:
         hidden_states = inputs_embeds
@@ -409,7 +403,7 @@ class FlashGemmaModel(torch.nn.Module):
                 kv_cache[i],
                 block_tables,
                 slots,
-                input_lengths,
+                seqlen,
                 max_s,
             )
 
@@ -419,11 +413,11 @@ class FlashGemmaModel(torch.nn.Module):
 
 
 class FlashGemmaForCausalLM(torch.nn.Module):
-    def __init__(self, prefix, config, weights, causal: bool):
+    def __init__(self, prefix: str, config, weights, *, causal: bool = True):
         super().__init__()
 
         embed_norm = config.hidden_size**0.5
-        if prefix is None:
+        if not prefix:
             prefix = "model"
         else:
             prefix = f"{prefix}.model"
@@ -454,9 +448,11 @@ class FlashGemmaForCausalLM(torch.nn.Module):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
+        prefill_cache_indices: Optional[torch.Tensor],
         lm_head_indices: Optional[torch.Tensor] = None,
+        adapter_data: Optional[torch.Tensor] = None,
     ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
         input_embeds = self.embed_tokens(input_ids)
         hidden_states = self.model(
@@ -466,7 +462,7 @@ class FlashGemmaForCausalLM(torch.nn.Module):
             kv_cache,
             block_tables,
             slots,
-            input_lengths,
+            seqlen,
             max_s,
         )
         if lm_head_indices is not None:
diff --git a/server/text_generation_server/models/custom_modeling/flash_gpt2_modeling.py b/server/text_generation_server/models/custom_modeling/flash_gpt2_modeling.py
index d2599f7a..44c015cf 100644
--- a/server/text_generation_server/models/custom_modeling/flash_gpt2_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/flash_gpt2_modeling.py
@@ -24,8 +24,13 @@ import torch.distributed
 from torch import nn
 from transformers.activations import ACT2FN
 from typing import Optional, List, Tuple
-
-from text_generation_server.utils import paged_attention, flash_attn
+from text_generation_server.layers.attention import PREFILL_IN_KV_CACHE
+from text_generation_server.layers.attention import (
+    paged_attention,
+    attention,
+    reshape_and_cache,
+    Seqlen,
+)
 from text_generation_server.layers import (
     TensorParallelRowLinear,
     TensorParallelColumnLinear,
@@ -42,6 +47,10 @@ def load_qkv(config, prefix: str, weights, head_size, num_heads):
             prefix,
             weights,
         )
+    elif config.quantize == "marlin":
+        raise RuntimeError(
+            "GPT-2 models with marlin quantization are not yet supported"
+        )
     else:
         return _load_qkv(config, prefix, weights, head_size, num_heads)
 
@@ -51,7 +60,11 @@ def _load_qkv_gptq(config, prefix: str, weights):
     rank = weights.process_group.rank()
 
     # Weights
-    weight = weights.get_weights_col_packed_qkv(f"{prefix}.c_attn", config.quantize)
+    weight = weights.get_weights_col_packed_qkv(
+        f"{prefix}.c_attn",
+        config.num_attention_heads,
+        config.num_attention_heads,
+    )
 
     # Bias
     slice_ = weights._get_slice(f"{prefix}.c_attn.bias")
@@ -70,7 +83,7 @@ def _load_qkv_gptq(config, prefix: str, weights):
     bias = torch.cat(tensors, dim=0)
     bias = bias.to(device=weights.device)
 
-    return TensorParallelColumnLinear(get_linear(weight, bias, config.quantize))
+    return TensorParallelColumnLinear(get_linear(weight, bias))
 
 
 def _load_qkv(config, prefix: str, weights, head_size, num_heads):
@@ -117,14 +130,14 @@ def _load_qkv(config, prefix: str, weights, head_size, num_heads):
         3 * num_heads * head_size
     ], f"{weight.shape} != {[3 * num_heads * head_size]}"
 
-    return TensorParallelColumnLinear(get_linear(weight, bias, config.quantize))
+    return TensorParallelColumnLinear(get_linear(weight, bias))
 
 
 def load_row(config, prefix: str, weights, bias: bool):
     """load_row, but with transposed weight matrices."""
 
     if config.quantize == "gptq":
-        weight = weights.get_multi_weights_row(prefix, quantize=config.quantize)
+        weight = weights.get_weights_row(prefix)
     else:
         weight = weights.get_sharded(f"{prefix}.weight", dim=0).T
 
@@ -135,16 +148,14 @@ def load_row(config, prefix: str, weights, bias: bool):
         bias = None
 
     return TensorParallelRowLinear(
-        get_linear(weight, bias, config.quantize), process_group=weights.process_group
+        get_linear(weight, bias), process_group=weights.process_group
     )
 
 
 def load_col(config, prefix: str, weights, bias: bool):
     """load_col, but with transposed weight matrices."""
     if config.quantize == "gptq":
-        weight = weights.get_multi_weights_col(
-            [prefix], quantize=config.quantize, dim=1
-        )
+        weight = weights.get_multi_weights_col([prefix], dim=1)
     else:
         weight = weights.get_sharded(f"{prefix}.weight", dim=1).T
 
@@ -153,7 +164,7 @@ def load_col(config, prefix: str, weights, bias: bool):
     else:
         bias = None
 
-    return TensorParallelColumnLinear(get_linear(weight, bias, config.quantize))
+    return TensorParallelColumnLinear(get_linear(weight, bias))
 
 
 class FlashGPT2Attention(torch.nn.Module):
@@ -203,7 +214,7 @@ class FlashGPT2Attention(torch.nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
     ):
         query, key, value = self.query_key_value(hidden_states).split(
@@ -213,34 +224,29 @@ class FlashGPT2Attention(torch.nn.Module):
         key = key.view(-1, self.num_heads, self.head_size)
         value = value.view(-1, self.num_heads, self.head_size)
 
-        paged_attention.reshape_and_cache(key, value, kv_cache[0], kv_cache[1], slots)
-
-        # output tensor
-        attn_output = torch.empty_like(query)
+        reshape_and_cache(key, value, kv_cache[0], kv_cache[1], slots)
 
         # Prefill
         if cu_seqlen_prefill is not None:
             # flash attention
-            flash_attn.attention(
+            attn_output = attention(
                 query,
-                key,
-                value,
-                attn_output,
-                cu_seqlen_prefill,
-                max_s,
+                kv_cache[0] if PREFILL_IN_KV_CACHE else key,
+                kv_cache[1] if PREFILL_IN_KV_CACHE else value,
+                seqlen,
+                block_tables,
                 self.softmax_scale,
             )
         # Decode
         else:
-            paged_attention.attention(
-                attn_output,
+            attn_output = paged_attention(
                 query,
                 kv_cache[0],
                 kv_cache[1],
                 self.kv_head_mapping,
                 self.softmax_scale,
                 block_tables,
-                input_lengths,
+                seqlen,
                 max_s,
             )
 
@@ -248,7 +254,7 @@ class FlashGPT2Attention(torch.nn.Module):
 
 
 class GPT2MLP(nn.Module):
-    def __init__(self, prefix, config, weights):
+    def __init__(self, prefix: str, config, weights):
         super().__init__()
         act = config.activation_function
         self.act = (
@@ -285,7 +291,7 @@ class GPT2MLP(nn.Module):
 
 
 class FlashGPT2Layer(nn.Module):
-    def __init__(self, prefix, config, weights):
+    def __init__(self, prefix: str, config, weights):
         super().__init__()
         self.self_attn = FlashGPT2Attention(
             prefix=f"{prefix}.attn", config=config, weights=weights
@@ -309,7 +315,7 @@ class FlashGPT2Layer(nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
     ):
         residual = hidden_states
@@ -322,7 +328,7 @@ class FlashGPT2Layer(nn.Module):
             kv_cache,
             block_tables,
             slots,
-            input_lengths,
+            seqlen,
             max_s,
         )
 
@@ -337,7 +343,7 @@ class FlashGPT2Layer(nn.Module):
 
 
 class FlashGPT2Model(torch.nn.Module):
-    def __init__(self, prefix, config, weights):
+    def __init__(self, prefix: str, config, weights):
         super().__init__()
 
         process_group = weights.process_group
@@ -375,7 +381,7 @@ class FlashGPT2Model(torch.nn.Module):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
         true_max_s: int,
         prefill_cache_indices: Optional[torch.Tensor],
@@ -391,7 +397,7 @@ class FlashGPT2Model(torch.nn.Module):
                 kv_cache[i],
                 block_tables,
                 slots,
-                input_lengths,
+                seqlen,
                 max_s,
             )
 
@@ -401,7 +407,7 @@ class FlashGPT2Model(torch.nn.Module):
 
 
 class FlashGPT2ForCausalLM(torch.nn.Module):
-    def __init__(self, prefix, config, weights):
+    def __init__(self, prefix: str, config, weights):
         super().__init__()
 
         self.embed_tokens = TensorParallelEmbedding(
@@ -428,10 +434,11 @@ class FlashGPT2ForCausalLM(torch.nn.Module):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
         prefill_cache_indices: Optional[torch.Tensor] = None,
         lm_head_indices: Optional[torch.Tensor] = None,
+        adapter_data: Optional[torch.Tensor] = None,
     ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
         token_embeds = self.embed_tokens(input_ids)
         position_embeds = self.embed_positions(position_ids)
@@ -443,7 +450,7 @@ class FlashGPT2ForCausalLM(torch.nn.Module):
             kv_cache,
             block_tables,
             slots,
-            input_lengths,
+            seqlen,
             max_s,
             true_max_s=max_s,
             prefill_cache_indices=prefill_cache_indices,
diff --git a/server/text_generation_server/models/custom_modeling/flash_gptj_modeling.py b/server/text_generation_server/models/custom_modeling/flash_gptj_modeling.py
new file mode 100644
index 00000000..aca97004
--- /dev/null
+++ b/server/text_generation_server/models/custom_modeling/flash_gptj_modeling.py
@@ -0,0 +1,406 @@
+# coding=utf-8
+# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
+#
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
+# and OPT implementations in this library. It has been modified from its
+# original forms to accommodate minor architectural differences compared
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import torch
+import torch.distributed
+
+from torch import nn
+from transformers.activations import ACT2FN
+from typing import Optional, List, Tuple
+from text_generation_server.utils.import_utils import SYSTEM
+from text_generation_server.layers.attention import (
+    paged_attention,
+    attention,
+    reshape_and_cache,
+    Seqlen,
+)
+from text_generation_server.layers import (
+    TensorParallelRowLinear,
+    TensorParallelColumnLinear,
+    TensorParallelEmbedding,
+    SpeculativeHead,
+    get_linear,
+)
+from text_generation_server.layers.attention import PREFILL_IN_KV_CACHE
+from text_generation_server.layers.rotary import (
+    PositionRotaryEmbedding,
+)
+from text_generation_server.layers.layernorm import (
+    FastLayerNorm,
+)
+
+
+def load_attention(config, prefix: str, weights):
+    return TensorParallelColumnLinear.load_multi(
+        config,
+        prefixes=[f"{prefix}.q_proj", f"{prefix}.k_proj", f"{prefix}.v_proj"],
+        dim=0,
+        weights=weights,
+        bias=False,
+    )
+
+
+def load_row(config, prefix: str, weights, bias: bool):
+    weight = weights.get_weights_row(prefix)
+
+    if bias and weights.process_group.rank() == 0:
+        # Rank is only on the first rank process
+        bias = weights.get_tensor(f"{prefix}.bias")
+    else:
+        bias = None
+
+    linear = get_linear(weight, bias)
+    return TensorParallelRowLinear(linear, process_group=weights.process_group)
+
+
+class GPTJRotary(PositionRotaryEmbedding):
+    def forward(
+        self,
+        query: torch.Tensor,
+        key: torch.Tensor,
+        cos: torch.Tensor,
+        sin: torch.Tensor,
+    ):
+        # Such controlflows may add some overhead.
+        if SYSTEM == "cuda":
+            import rotary_emb
+
+            q1 = query[..., ::2]
+            q2 = query[..., 1::2]
+
+            rotary_emb.apply_rotary(q1, q2, cos, sin, q1, q2, False)
+
+            k1 = key[..., ::2]
+            k2 = key[..., 1::2]
+
+            rotary_emb.apply_rotary(k1, k2, cos, sin, k1, k2, False)
+        elif SYSTEM == "rocm":
+            from vllm._C import ops
+
+            # NOTE: On RoCm systems, we use a ROPE implementatation adapted from VLLM which launches a single kernel for both query/key, contrary to flash-attn implementation used on NVIDIA systems.
+            # Compiling flash-attn rotary on RoCm, it appears hipcc is unable to unroll loops, resulting in an even slower inference compared to eager: https://github.com/pytorch/pytorch/issues/113773
+
+            head_size = query.shape[-1]
+
+            # Inplace operation, updating query and key.
+            ops.rotary_embedding(query, key, head_size, cos, sin, False)
+        elif SYSTEM == "ipex":
+            import intel_extension_for_pytorch as ipex
+
+            ipex.llm.functional.rotary_embedding(
+                query, key, sin, cos, query.size(-1), False
+            )
+        else:
+            raise ValueError(
+                "Your system seem to be not supported. Please check your install or open an issue at https://github.com/huggingface/text-generation-inference/issues with a clear reproduction."
+            )
+
+
+class FlashGPTJAttention(torch.nn.Module):
+    def __init__(
+        self,
+        prefix: str,
+        config,
+        weights,
+    ):
+        super().__init__()
+        self.num_heads = config.num_attention_heads
+        self.hidden_size = config.hidden_size
+
+        self.head_size = self.hidden_size // self.num_heads
+        self.softmax_scale = self.head_size**-0.5
+        self.rotary_dim = config.rotary_dim
+
+        if self.num_heads % weights.process_group.size() != 0:
+            raise ValueError(
+                f"`num_heads` must be divisible by `num_shards` (got `num_heads`: {self.num_heads} "
+                f"and `num_shards`: {weights.process_group.size()}"
+            )
+        self.num_heads = self.num_heads // weights.process_group.size()
+
+        self.query_key_value = load_attention(
+            config,
+            prefix=prefix,
+            weights=weights,
+        )
+
+        self.o_proj = load_row(
+            config,
+            prefix=f"{prefix}.out_proj",
+            weights=weights,
+            bias=False,
+        )
+
+        self.kv_head_mapping = torch.arange(
+            0, self.num_heads, dtype=torch.int32, device=weights.device
+        )
+
+        self.rotary_emb = GPTJRotary.static(
+            config=config,
+            dim=self.rotary_dim,
+            base=10000,
+            device=weights.device,
+        )
+
+    def forward(
+        self,
+        hidden_states,
+        cos,
+        sin,
+        cu_seqlen_prefill,
+        kv_cache,
+        block_tables,
+        slots,
+        seqlen,
+        max_s,
+    ):
+        query, key, value = self.query_key_value(hidden_states).split(
+            self.head_size * self.num_heads, dim=1
+        )
+        query = query.view(-1, self.num_heads, self.head_size)
+        key = key.view(-1, self.num_heads, self.head_size)
+        value = value.view(-1, self.num_heads, self.head_size)
+
+        # Compute rotary embeddings on rotary_ndims
+        if self.rotary_dim is not None:
+            self.rotary_emb(
+                query[..., : self.rotary_dim], key[..., : self.rotary_dim], cos, sin
+            )
+        else:
+            self.rotary_emb(query, key, cos, sin)
+
+        reshape_and_cache(key, value, kv_cache[0], kv_cache[1], slots)
+
+        # Prefill
+        if cu_seqlen_prefill is not None:
+            # flash attention
+            attn_output = attention(
+                query,
+                kv_cache[0] if PREFILL_IN_KV_CACHE else key,
+                kv_cache[1] if PREFILL_IN_KV_CACHE else value,
+                seqlen,
+                block_tables,
+                self.softmax_scale,
+            )
+        # Decode
+        else:
+            attn_output = paged_attention(
+                query,
+                kv_cache[0],
+                kv_cache[1],
+                self.kv_head_mapping,
+                self.softmax_scale,
+                block_tables,
+                seqlen,
+                max_s,
+            )
+
+        return self.o_proj(attn_output.view(-1, self.num_heads * self.head_size))
+
+
+class GPTJMLP(nn.Module):
+    def __init__(self, prefix: str, config, weights):
+        super().__init__()
+        act = config.activation_function
+        self.act = (
+            ACT2FN[act]
+            if "gelu" not in act
+            else lambda x: torch.nn.functional.gelu(
+                x,
+                approximate=(
+                    "tanh" if act in ["gelu_fast", "gelu_pytorch_tanh"] else "none"
+                ),
+            )
+        )
+
+        self.fc_in = TensorParallelColumnLinear.load(
+            config, prefix=f"{prefix}.fc_in", weights=weights, bias=True
+        )
+
+        self.fc_out = load_row(
+            config,
+            prefix=f"{prefix}.fc_out",
+            weights=weights,
+            bias=True,
+        )
+
+    def forward(self, hidden_states):
+        hidden_states = self.fc_in(hidden_states)
+        hidden_states = self.act(hidden_states)
+        return self.fc_out(hidden_states)
+
+
+class FlashGPTJLayer(nn.Module):
+    def __init__(self, prefix: str, config, weights):
+        super().__init__()
+        self.self_attn = FlashGPTJAttention(
+            prefix=f"{prefix}.attn", config=config, weights=weights
+        )
+        self.mlp = GPTJMLP(prefix=f"{prefix}.mlp", config=config, weights=weights)
+
+        self.input_layernorm = FastLayerNorm.load(
+            prefix=f"{prefix}.ln_1", weights=weights, eps=config.layer_norm_epsilon
+        )
+
+    def forward(
+        self,
+        hidden_states,
+        residual,
+        cos,
+        sin,
+        cu_seqlen_prefill,
+        kv_cache,
+        block_tables,
+        slots,
+        seqlen,
+        max_s,
+    ):
+        hidden_states, residual = self.input_layernorm(hidden_states, residual)
+        # Self Attention
+        attn_output = self.self_attn(
+            hidden_states,
+            cos,
+            sin,
+            cu_seqlen_prefill,
+            kv_cache,
+            block_tables,
+            slots,
+            seqlen,
+            max_s,
+        )
+
+        feed_forward_hidden_states = self.mlp(hidden_states)
+
+        return attn_output + feed_forward_hidden_states, residual
+
+
+class FlashGPTJModel(torch.nn.Module):
+    def __init__(self, prefix: str, config, weights):
+        super().__init__()
+        self.config = config
+
+        self.wte = TensorParallelEmbedding(prefix=f"{prefix}.wte", weights=weights)
+        self.layers = nn.ModuleList(
+            [
+                FlashGPTJLayer(
+                    prefix=(
+                        f"h.{layer_id}" if not prefix else f"{prefix}.h.{layer_id}"
+                    ),
+                    config=config,
+                    weights=weights,
+                )
+                for layer_id in range(config.num_hidden_layers)
+            ]
+        )
+
+        self.ln_f = FastLayerNorm.load(
+            prefix="ln_f" if not prefix else f"{prefix}.ln_f",
+            weights=weights,
+            eps=config.layer_norm_epsilon,
+        )
+
+        self.gradient_checkpointing = False
+
+        self.head_size = self.layers[0].self_attn.head_size
+        self.num_heads = self.layers[0].self_attn.num_heads
+
+    def forward(
+        self,
+        input_ids: Optional[torch.LongTensor],
+        position_ids: torch.Tensor,
+        cu_seqlen_prefill: Optional[torch.Tensor],
+        kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
+        block_tables: torch.Tensor,
+        slots: torch.Tensor,
+        seqlen: Seqlen,
+        max_s: int,
+        prefill_cache_indices: Optional[torch.Tensor],
+    ) -> torch.Tensor:
+        hidden_states = self.wte(input_ids)
+
+        # Get rotary cos and sin for this forward
+        # Avoid to index in each layer
+        cos, sin = self.layers[0].self_attn.rotary_emb.get_cos_sin(
+            position_ids, max_s, hidden_states.dtype
+        )
+
+        residual = None
+        for i, layer in enumerate(self.layers):
+            hidden_states, residual = layer(
+                hidden_states,
+                residual,
+                cos,
+                sin,
+                cu_seqlen_prefill,
+                kv_cache[i],
+                block_tables,
+                slots,
+                seqlen,
+                max_s,
+            )
+
+        hidden_states, _ = self.ln_f(hidden_states, residual)
+
+        return hidden_states
+
+
+class FlashGPTJForCausalLM(torch.nn.Module):
+    def __init__(self, prefix: str, config, weights):
+        super().__init__()
+        if not prefix:
+            prefix = "transformer"
+        else:
+            prefix = f"{prefix}.transformer"
+        self.model = FlashGPTJModel(prefix, config, weights)
+        self.lm_head = SpeculativeHead.load(
+            config,
+            prefix="lm_head",
+            weights=weights,
+        )
+
+    def forward(
+        self,
+        input_ids: torch.Tensor,
+        position_ids: torch.Tensor,
+        cu_seqlen_prefill: Optional[torch.Tensor],
+        kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
+        block_tables: torch.Tensor,
+        slots: torch.Tensor,
+        seqlen: Seqlen,
+        max_s: int,
+        prefill_cache_indices: Optional[torch.Tensor] = None,
+        lm_head_indices: Optional[torch.Tensor] = None,
+        adapter_data: Optional[torch.Tensor] = None,
+    ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
+        hidden_states = self.model(
+            input_ids,
+            position_ids,
+            cu_seqlen_prefill,
+            kv_cache,
+            block_tables,
+            slots,
+            seqlen,
+            max_s,
+            prefill_cache_indices=prefill_cache_indices,
+        )
+        if lm_head_indices is not None:
+            hidden_states = hidden_states[lm_head_indices]
+        logits, speculative_logits = self.lm_head(hidden_states)
+        return logits, speculative_logits
diff --git a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py
index 6e23aa2b..c9ec70cc 100644
--- a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py
@@ -18,27 +18,47 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import List, Optional, Tuple
+from contextlib import contextmanager
+from typing import List, Optional, Tuple, Type
 
 import torch
 import torch.distributed
 
 from torch import nn
 from transformers.activations import ACT2FN
-from typing import Optional, List, Tuple
 
+from text_generation_server.layers.attention import PREFILL_IN_KV_CACHE
+from text_generation_server.layers.moe import DenseMoELayer, MoELayer, SparseMoELayer
 from text_generation_server.utils.import_utils import SYSTEM
-from text_generation_server.utils import paged_attention, flash_attn
+from text_generation_server.layers.attention import (
+    paged_attention,
+    attention,
+    reshape_and_cache,
+    Seqlen,
+)
 from text_generation_server.layers import (
     TensorParallelRowLinear,
     TensorParallelColumnLinear,
     TensorParallelEmbedding,
     SpeculativeHead,
+    TensorParallelMultiAdapterLinear,
+    TensorParallelAdapterRowLinear,
 )
 from text_generation_server.layers.rotary import PositionRotaryEmbedding
 from text_generation_server.layers.layernorm import (
     FastRMSNorm,
+    FastLayerNorm,
 )
+from text_generation_server.layers import (
+    FastLinear,
+)
+from text_generation_server.utils.weights import (
+    Weights,
+)
+from text_generation_server.layers.fp8 import HybridFP8UnquantLoader
+
+if SYSTEM != "ipex":
+    pass
 
 if SYSTEM == "rocm":
     try:
@@ -47,44 +67,75 @@ if SYSTEM == "rocm":
         raise ImportError(f"Could not load `vllm._custom_C`. Full error: {e}")
 
 
-def load_attention(config, prefix, weights):
-    bias = config.attention_bias
-    if config.num_attention_heads != config.num_key_value_heads:
-        return TensorParallelColumnLinear.load_multi(
+def load_attention(config, prefix: str, weights, layer_id):
+    # Only defined in granite.
+    bias = getattr(config, "attention_bias", False)
+    head_size = config.hidden_size // config.num_attention_heads
+    sizes = None
+    prefixes = None
+
+    if config.model_type == "phi3":
+        base_layer = TensorParallelColumnLinear.load_qkv(
+            config,
+            prefix=f"{prefix}.qkv_proj",
+            weights=weights,
+            bias=bias,
+            num_heads=config.num_attention_heads,
+            num_key_value_heads=config.num_key_value_heads,
+        )
+        prefixes = ["qkv_proj"]
+    elif config.model_type == "baichuan":
+        prefix = f"{prefix}.W_pack"
+        base_layer = TensorParallelColumnLinear.load_qkv(
+            config,
+            prefix=prefix,
+            weights=weights,
+            bias=bias,
+            num_heads=config.num_attention_heads,
+            num_key_value_heads=config.num_key_value_heads,
+        )
+        prefixes = [prefix]
+    else:
+        prefixes = ["q_proj", "k_proj", "v_proj"]
+        sizes = [
+            head_size * config.num_attention_heads,
+            head_size * config.num_key_value_heads,
+            head_size * config.num_key_value_heads,
+        ]
+        base_layer = TensorParallelColumnLinear.load_multi(
             config,
             prefixes=[f"{prefix}.q_proj", f"{prefix}.k_proj", f"{prefix}.v_proj"],
             dim=0,
             weights=weights,
             bias=bias,
         )
-    else:
-        if config.model_type == "baichuan":
-            return TensorParallelColumnLinear.load_qkv(
-                config,
-                prefix=f"{prefix}.W_pack",
-                weights=weights,
-                bias=bias,
-            )
-        elif config.model_type == "phi3":
-            return TensorParallelColumnLinear.load_qkv(
-                config,
-                prefix=f"{prefix}.qkv_proj",
-                weights=weights,
-                bias=bias,
-            )
-        else:
-            return TensorParallelColumnLinear.load_multi(
-                config,
-                prefixes=[f"{prefix}.q_proj", f"{prefix}.k_proj", f"{prefix}.v_proj"],
-                dim=0,
-                weights=weights,
-                bias=bias,
-            )
+
+    return TensorParallelMultiAdapterLinear.load(
+        base_layer=base_layer,
+        layer_id=layer_id,
+        layer_names=prefixes,
+        sizes=sizes,
+        process_group=weights.process_group,
+    )
+
+
+@contextmanager
+def no_fp8(weights: Weights):
+    """De-activate fp8 auto conversion for the duration of this context manager"""
+    weights_loader = weights.weights_loader
+    if isinstance(weights_loader, HybridFP8UnquantLoader) and weights_loader.to_fp8:
+        weights_loader = HybridFP8UnquantLoader(
+            weights_loader.activation_scale_ub, to_fp8=False
+        )
+
+    with weights.use_loader(weights_loader):
+        yield
 
 
 class FlashLlamaAttention(torch.nn.Module):
     def __init__(
         self,
+        index: int,
         prefix: str,
         config,
         weights,
@@ -94,6 +145,11 @@ class FlashLlamaAttention(torch.nn.Module):
         self.hidden_size = config.hidden_size
         self.head_size = self.hidden_size // self.num_heads
 
+        # Setting defaults for baichuan custom config which doesn't apply them.
+        config.rope_theta = getattr(config, "rope_theta", 10000)
+        config.num_key_value_heads = getattr(
+            config, "num_key_value_heads", config.num_attention_heads
+        )
         self.rotary_emb = PositionRotaryEmbedding.static(
             config=config,
             dim=self.head_size,
@@ -108,19 +164,33 @@ class FlashLlamaAttention(torch.nn.Module):
                 f"`num_heads` must be divisible by `num_shards` (got `num_heads`: {self.num_heads} "
                 f"and `num_shards`: {weights.process_group.size()}"
             )
+        if config.num_key_value_heads % weights.process_group.size() != 0:
+            raise ValueError(
+                f"`num_key_value_heads` must be divisible by `num_shards` (got `num_key_value_heads`: {config.num_key_value_heads} "
+                f"and `num_shards`: {weights.process_group.size()}"
+            )
         self.num_heads = self.num_heads // weights.process_group.size()
         self.num_key_value_heads = (
             config.num_key_value_heads // weights.process_group.size()
         )
 
-        self.query_key_value = load_attention(config, prefix, weights)
+        self.query_key_value = load_attention(config, prefix, weights, index)
+        self.index = index
 
-        self.o_proj = TensorParallelRowLinear.load(
+        o_proj = TensorParallelRowLinear.load(
             config,
             prefix=f"{prefix}.o_proj",
             weights=weights,
             bias=False,
         )
+
+        self.o_proj = TensorParallelAdapterRowLinear.load(
+            o_proj,
+            index,
+            "o_proj",
+            process_group=weights.process_group,
+        )
+
         self.num_groups = self.num_heads // self.num_key_value_heads
         self.kv_head_mapping = torch.arange(
             0, self.num_key_value_heads, dtype=torch.int32, device=weights.device
@@ -135,10 +205,11 @@ class FlashLlamaAttention(torch.nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
+        adapter_data,
     ):
-        qkv = self.query_key_value(hidden_states)
+        qkv = self.query_key_value(hidden_states, adapter_data)
         query, kv = qkv.split(
             [
                 self.head_size * self.num_heads,
@@ -151,44 +222,75 @@ class FlashLlamaAttention(torch.nn.Module):
 
         self.rotary_emb(query, torch.select(kv, dim=1, index=0), cos, sin)
 
-        paged_attention.reshape_and_cache(
-            kv[:, 0], kv[:, 1], kv_cache[0], kv_cache[1], slots
-        )
-
-        # output tensor
-        attn_output = torch.empty_like(query)
+        reshape_and_cache(kv[:, 0], kv[:, 1], kv_cache[0], kv_cache[1], slots)
 
         # Prefill
         if cu_seqlen_prefill is not None:
             # flash attention
-            flash_attn.attention(
+            attn_output = attention(
                 query,
-                torch.select(kv, dim=1, index=0),
-                torch.select(kv, dim=1, index=1),
-                attn_output,
-                cu_seqlen_prefill,
-                max_s,
+                kv_cache[0] if PREFILL_IN_KV_CACHE else kv[:, 0],
+                kv_cache[1] if PREFILL_IN_KV_CACHE else kv[:, 1],
+                seqlen,
+                block_tables,
                 self.softmax_scale,
             )
         # Decode
         else:
-            paged_attention.attention(
-                attn_output,
+            attn_output = paged_attention(
                 query,
                 kv_cache[0],
                 kv_cache[1],
                 self.kv_head_mapping,
                 self.softmax_scale,
                 block_tables,
-                input_lengths,
+                seqlen,
                 max_s,
             )
 
-        return self.o_proj(attn_output.view(-1, self.num_heads * self.head_size))
+        return self.o_proj(
+            attn_output.view(-1, self.num_heads * self.head_size), adapter_data
+        )
+
+
+class Phi3MoE(nn.Module):
+    def __init__(
+        self, prefix: str, config, moe_layer_cls: Type[MoELayer], weights: Weights
+    ):
+        super().__init__()
+
+        # gating
+        self.gate = FastLinear.load(config, f"{prefix}.gate", weights, bias=False)
+
+        self.moe = moe_layer_cls(
+            prefix=f"{prefix}.experts",
+            n_experts=config.num_local_experts,
+            n_expert_group=None,
+            renormalize=True,
+            topk=config.num_experts_per_tok,
+            topk_group=None,
+            weights=weights,
+            gate_proj_name="w1",
+            up_proj_name="w3",
+            down_proj_name="w2",
+        )
+
+        self.process_group = weights.process_group
+
+    def forward(self, x, adapter_data) -> torch.Tensor:
+        # router_logits: (num_tokens, n_experts)
+        router_logits = self.gate(x)
+        out = self.moe(x, gating_output=router_logits)
+
+        # Reduce sum
+        if self.process_group.size() > 1:
+            torch.distributed.all_reduce(out, group=self.process_group)
+
+        return out.view(*x.shape)
 
 
 class LlamaMLP(nn.Module):
-    def __init__(self, prefix, config, weights):
+    def __init__(self, prefix, config, weights, index):
         super().__init__()
         self.hidden_act = config.hidden_act
         self.act = (
@@ -203,29 +305,54 @@ class LlamaMLP(nn.Module):
                 ),
             )
         )
+        prefixes = None
+        sizes = None
+
         # Fuse gate and up proj
         bias = getattr(config, "mlp_bias", False)
         if config.model_type == "phi3":
-            self.gate_up_proj = TensorParallelColumnLinear.load_gate_up(
+            gate_up_proj = TensorParallelColumnLinear.load_gate_up(
                 config,
                 prefix=f"{prefix}.gate_up_proj",
                 weights=weights,
                 bias=bias,
             )
         else:
-            self.gate_up_proj = TensorParallelColumnLinear.load_multi(
+            prefixes = ["gate_proj", "up_proj"]
+            sizes = [
+                config.intermediate_size,
+                config.intermediate_size,
+            ]
+            gate_up_proj = TensorParallelColumnLinear.load_multi(
                 config,
                 prefixes=[f"{prefix}.gate_proj", f"{prefix}.up_proj"],
                 weights=weights,
                 dim=0,
                 bias=bias,
             )
-        self.down_proj = TensorParallelRowLinear.load(
+
+        self.gate_up_proj = TensorParallelMultiAdapterLinear.load(
+            gate_up_proj,
+            index,
+            layer_names=prefixes,
+            sizes=sizes,
+            process_group=weights.process_group,
+        )
+
+        down_proj = TensorParallelRowLinear.load(
             config,
             prefix=f"{prefix}.down_proj",
             weights=weights,
             bias=bias,
         )
+
+        self.down_proj = TensorParallelAdapterRowLinear.load(
+            down_proj,
+            index,
+            "down_proj",
+            process_group=weights.process_group,
+        )
+
         self.intermediate_size = (
             config.intermediate_size // weights.process_group.size()
         )
@@ -233,12 +360,17 @@ class LlamaMLP(nn.Module):
         # TODO: This is a hotfix to be removed & properly refactored.
         self.quantize = config.quantize
 
-    def forward(self, hidden_states):
+        self.hidden_size = config.hidden_size
+
+    def forward(self, hidden_states, adapter_data):
         if (
             SYSTEM == "rocm"
             and self.hidden_act == "silu"
+            and hidden_states.dtype == torch.float16
             and hidden_states.shape[0] == 1
             and not self.quantize
+            and self.hidden_size
+            != 16384  # TODO: Temporary workaround for `LLMM_Silu` kernel not working with LLama3.1 405B; needs refactoring once fixed.
         ):
             out = torch.empty(
                 hidden_states.shape[0],
@@ -246,30 +378,64 @@ class LlamaMLP(nn.Module):
                 dtype=hidden_states.dtype,
                 device="cuda",
             )
-            _custom_C.LLMM_Silu(self.gate_up_proj.linear.weight, hidden_states, out, 8)
-            return self.down_proj(out)
+            _custom_C.LLMM_Silu(
+                self.gate_up_proj.base_layer.linear.weight, hidden_states, out, 8
+            )
+            return self.down_proj(out, adapter_data)
         else:
-            gate_up_states = self.gate_up_proj(hidden_states)
+            gate_up_states = self.gate_up_proj(hidden_states, adapter_data)
             gate_up_states = gate_up_states.view(-1, 2, self.intermediate_size)
-            return self.down_proj(self.act(gate_up_states[:, 0]) * gate_up_states[:, 1])
+            return self.down_proj(
+                self.act(gate_up_states[:, 0]) * gate_up_states[:, 1], adapter_data
+            )
 
 
 class FlashLlamaLayer(nn.Module):
-    def __init__(self, prefix, config, weights):
+    def __init__(self, index, prefix, config, weights):
         super().__init__()
-        self.self_attn = FlashLlamaAttention(
-            prefix=f"{prefix}.self_attn", config=config, weights=weights
-        )
-        self.mlp = LlamaMLP(prefix=f"{prefix}.mlp", config=config, weights=weights)
 
-        self.input_layernorm = FastRMSNorm.load(
-            prefix=f"{prefix}.input_layernorm", weights=weights, eps=config.rms_norm_eps
-        )
-        self.post_attention_layernorm = FastRMSNorm.load(
-            prefix=f"{prefix}.post_attention_layernorm",
-            weights=weights,
-            eps=config.rms_norm_eps,
-        )
+        with no_fp8(weights):
+            self.self_attn = FlashLlamaAttention(
+                index=index,
+                prefix=f"{prefix}.self_attn",
+                config=config,
+                weights=weights,
+            )
+
+        if config.model_type == "phimoe":
+            moe_layer_cls = (
+                SparseMoELayer
+                if SparseMoELayer.is_supported(weights)
+                else DenseMoELayer
+            )
+            self.dense = Phi3MoE(
+                f"{prefix}.block_sparse_moe", config, moe_layer_cls, weights
+            )
+            # with moe the layernorms are are not rmsnorms and they have bias
+            self.input_layernorm = FastLayerNorm.load(
+                prefix=f"{prefix}.input_layernorm",
+                weights=weights,
+                eps=config.rms_norm_eps,
+            )
+            self.post_attention_layernorm = FastLayerNorm.load(
+                prefix=f"{prefix}.post_attention_layernorm",
+                weights=weights,
+                eps=config.rms_norm_eps,
+            )
+        else:
+            self.dense = LlamaMLP(
+                prefix=f"{prefix}.mlp", config=config, weights=weights, index=index
+            )
+            self.input_layernorm = FastRMSNorm.load(
+                prefix=f"{prefix}.input_layernorm",
+                weights=weights,
+                eps=config.rms_norm_eps,
+            )
+            self.post_attention_layernorm = FastRMSNorm.load(
+                prefix=f"{prefix}.post_attention_layernorm",
+                weights=weights,
+                eps=config.rms_norm_eps,
+            )
 
     def forward(
         self,
@@ -281,8 +447,10 @@ class FlashLlamaLayer(nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
+        adapter_data,
+        cross_attention_states,
     ):
         normed_hidden_states, res = self.input_layernorm(hidden_states, residual)
 
@@ -295,8 +463,9 @@ class FlashLlamaLayer(nn.Module):
             kv_cache,
             block_tables,
             slots,
-            input_lengths,
+            seqlen,
             max_s,
+            adapter_data,
         )
 
         # faster post attention rms norm
@@ -304,7 +473,7 @@ class FlashLlamaLayer(nn.Module):
             attn_output, res
         )
 
-        mlp_output = self.mlp(normed_attn_res_output)
+        mlp_output = self.dense(normed_attn_res_output, adapter_data)
 
         return mlp_output, attn_res
 
@@ -316,20 +485,70 @@ class FlashLlamaModel(torch.nn.Module):
         process_group = weights.process_group
         self.tp_rank = process_group.rank()
         self.tp_world_size = process_group.size()
-        self.layers = nn.ModuleList(
-            [
+
+        # Skip fp8 quant for first and last layers
+        self.layers = nn.ModuleList()
+        self.cross_attention_layers = getattr(config, "cross_attention_layers", [])
+        with no_fp8(weights):
+            self.layers.append(
                 FlashLlamaLayer(
+                    index=0,
                     prefix=(
-                        f"model.layers.{layer_id}"
-                        if not prefix
-                        else f"{prefix}.model.layers.{layer_id}"
+                        "model.layers.0" if not prefix else f"{prefix}.model.layers.0"
                     ),
                     config=config,
                     weights=weights,
                 )
-                for layer_id in range(config.num_hidden_layers)
-            ]
-        )
+            )
+
+        # Skip first and last layers
+        for layer_id in range(1, config.num_hidden_layers - 1):
+            if layer_id in self.cross_attention_layers:
+                from text_generation_server.models.custom_modeling.mllama import (
+                    FlashLlamaCrossLayer,
+                )
+
+                self.layers.append(
+                    FlashLlamaCrossLayer(
+                        index=layer_id,
+                        prefix=(
+                            f"model.layers.{layer_id}"
+                            if not prefix
+                            else f"{prefix}.model.layers.{layer_id}"
+                        ),
+                        config=config,
+                        weights=weights,
+                    )
+                )
+            else:
+                self.layers.append(
+                    FlashLlamaLayer(
+                        index=layer_id,
+                        prefix=(
+                            f"model.layers.{layer_id}"
+                            if not prefix
+                            else f"{prefix}.model.layers.{layer_id}"
+                        ),
+                        config=config,
+                        weights=weights,
+                    )
+                )
+
+        with no_fp8(weights):
+            last_layer_id = config.num_hidden_layers - 1
+            self.layers.append(
+                FlashLlamaLayer(
+                    index=last_layer_id,
+                    prefix=(
+                        f"model.layers.{last_layer_id}"
+                        if not prefix
+                        else f"{prefix}.model.layers.{last_layer_id}"
+                    ),
+                    config=config,
+                    weights=weights,
+                )
+            )
+
         self.norm = FastRMSNorm.load(
             prefix="model.norm" if not prefix else f"{prefix}.model.norm",
             weights=weights,
@@ -350,10 +569,12 @@ class FlashLlamaModel(torch.nn.Module):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
         true_max_s: int,
         prefill_cache_indices: Optional[torch.Tensor],
+        adapter_data,
+        cross_attention_states=None,
     ) -> torch.Tensor:
         hidden_states = inputs_embeds
 
@@ -374,8 +595,10 @@ class FlashLlamaModel(torch.nn.Module):
                 kv_cache[i],
                 block_tables,
                 slots,
-                input_lengths,
+                seqlen,
                 max_s,
+                adapter_data,
+                cross_attention_states,
             )
 
         hidden_states, _ = self.norm(hidden_states, residual)
@@ -384,26 +607,30 @@ class FlashLlamaModel(torch.nn.Module):
 
 
 class FlashLlamaForCausalLM(torch.nn.Module):
-    def __init__(self, prefix, config, weights):
+    def __init__(self, prefix: str, config, weights):
         super().__init__()
 
-        self.embed_tokens = TensorParallelEmbedding(
-            prefix=(
-                "model.embed_tokens" if not prefix else f"{prefix}.model.embed_tokens"
-            ),
-            weights=weights,
-        )
+        with no_fp8(weights):
+            self.embed_tokens = TensorParallelEmbedding(
+                prefix=(
+                    "model.embed_tokens"
+                    if not prefix
+                    else f"{prefix}.model.embed_tokens"
+                ),
+                weights=weights,
+            )
         self.model = FlashLlamaModel(prefix, config, weights)
         if config.tie_word_embeddings:
             suffix = "model.embed_tokens"
         else:
             suffix = "lm_head"
 
-        self.lm_head = SpeculativeHead.load(
-            config,
-            prefix=suffix if not prefix else f"{prefix}.suffix",
-            weights=weights,
-        )
+        with no_fp8(weights):
+            self.lm_head = SpeculativeHead.load(
+                config,
+                prefix=suffix if not prefix else f"{prefix}.{suffix}",
+                weights=weights,
+            )
 
     def forward(
         self,
@@ -413,10 +640,12 @@ class FlashLlamaForCausalLM(torch.nn.Module):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
         prefill_cache_indices: Optional[torch.Tensor] = None,
         lm_head_indices: Optional[torch.Tensor] = None,
+        adapter_data: Optional[torch.Tensor] = None,
+        cross_attention_states=None,
     ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
         inputs_embeds = self.embed_tokens(input_ids)
         hidden_states = self.model(
@@ -426,10 +655,12 @@ class FlashLlamaForCausalLM(torch.nn.Module):
             kv_cache,
             block_tables,
             slots,
-            input_lengths,
+            seqlen,
             max_s,
             true_max_s=max_s,
             prefill_cache_indices=prefill_cache_indices,
+            adapter_data=adapter_data,
+            cross_attention_states=cross_attention_states,
         )
         if lm_head_indices is not None:
             hidden_states = hidden_states[lm_head_indices]
diff --git a/server/text_generation_server/models/custom_modeling/flash_mistral_modeling.py b/server/text_generation_server/models/custom_modeling/flash_mistral_modeling.py
index ef3777da..341a2352 100644
--- a/server/text_generation_server/models/custom_modeling/flash_mistral_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/flash_mistral_modeling.py
@@ -27,14 +27,21 @@ from transformers.configuration_utils import PretrainedConfig
 from typing import Optional, List, Tuple
 
 from text_generation_server.utils.import_utils import SYSTEM
-from text_generation_server.utils import paged_attention, flash_attn
+from text_generation_server.layers.attention import (
+    paged_attention,
+    attention,
+    reshape_and_cache,
+    Seqlen,
+)
 from text_generation_server.layers import (
     TensorParallelRowLinear,
     TensorParallelColumnLinear,
     TensorParallelEmbedding,
     SpeculativeHead,
-    get_linear,
+    TensorParallelMultiAdapterLinear,
+    TensorParallelAdapterRowLinear,
 )
+from text_generation_server.layers.attention import PREFILL_IN_KV_CACHE
 from text_generation_server.layers.rotary import PositionRotaryEmbedding
 from text_generation_server.layers.layernorm import (
     FastRMSNorm,
@@ -102,59 +109,18 @@ class MistralConfig(PretrainedConfig):
         )
 
 
-def load_attention(config, prefix, weights):
-    if config.num_attention_heads != config.num_key_value_heads:
-        return _load_gqa(config, prefix, weights)
-    else:
-        return TensorParallelColumnLinear.load_multi(
-            config,
-            prefixes=[f"{prefix}.q_proj", f"{prefix}.k_proj", f"{prefix}.v_proj"],
-            dim=0,
-            weights=weights,
-            bias=False,
-        )
-
-
-def _load_gqa(config, prefix: str, weights):
-    assert config.hidden_size % config.num_attention_heads == 0
-    assert config.num_attention_heads % weights.process_group.size() == 0
-
-    weight = weights.get_multi_weights_col(
-        prefixes=[f"{prefix}.q_proj", f"{prefix}.k_proj", f"{prefix}.v_proj"],
-        quantize=config.quantize,
-        dim=0,
-    )
-
-    if config.quantize not in ["gptq", "awq"]:
-        weight = weight.to(dtype=weights.dtype).to(device=weights.device)
-
-        head_size = config.hidden_size // config.num_attention_heads
-        num_heads = config.num_attention_heads // weights.process_group.size()
-        num_key_value_heads = config.num_key_value_heads // weights.process_group.size()
-        assert list(weight.shape) == [
-            (num_heads + 2 * num_key_value_heads) * head_size,
-            config.hidden_size,
-        ], f"{list(weight.shape)} != {[(num_heads + 2 * config.num_key_value_heads) * head_size, config.hidden_size]}"
-
-    return TensorParallelColumnLinear(
-        get_linear(weight, bias=None, quantize=config.quantize)
-    )
-
-
 class MistralAttention(torch.nn.Module):
-    def __init__(
-        self,
-        prefix: str,
-        config,
-        weights,
-    ):
+    def __init__(self, prefix: str, config, weights, layer_id):
         super().__init__()
         self.max_past = (
             config.sliding_window if config.sliding_window is not None else -1
         )
         self.num_heads = config.num_attention_heads
         self.hidden_size = config.hidden_size
-        self.head_size = self.hidden_size // self.num_heads
+        if hasattr(config, "head_dim"):
+            self.head_size = config.head_dim
+        else:
+            self.head_size = self.hidden_size // self.num_heads
 
         self.rotary_emb = PositionRotaryEmbedding.static(
             config=config,
@@ -175,14 +141,38 @@ class MistralAttention(torch.nn.Module):
             config.num_key_value_heads // weights.process_group.size()
         )
 
-        self.query_key_value = load_attention(config, prefix, weights)
+        query_key_value = TensorParallelColumnLinear.load_multi(
+            config,
+            prefixes=[f"{prefix}.q_proj", f"{prefix}.k_proj", f"{prefix}.v_proj"],
+            dim=0,
+            weights=weights,
+            bias=False,
+        )
 
-        self.o_proj = TensorParallelRowLinear.load(
+        self.query_key_value = TensorParallelMultiAdapterLinear.load(
+            query_key_value,
+            layer_id,
+            ["q_proj", "k_proj", "v_proj"],
+            sizes=[
+                self.head_size * config.num_attention_heads,
+                self.head_size * config.num_key_value_heads,
+                self.head_size * config.num_key_value_heads,
+            ],
+            process_group=weights.process_group,
+        )
+
+        o_proj = TensorParallelRowLinear.load(
             config,
             prefix=f"{prefix}.o_proj",
             weights=weights,
             bias=False,
         )
+        self.o_proj = TensorParallelAdapterRowLinear.load(
+            o_proj,
+            layer_id,
+            "o_proj",
+            process_group=weights.process_group,
+        )
         self.num_groups = self.num_heads // self.num_key_value_heads
         self.kv_head_mapping = torch.arange(
             0, self.num_key_value_heads, dtype=torch.int32, device=weights.device
@@ -197,11 +187,12 @@ class MistralAttention(torch.nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
         prefill_cache_indices,
+        adapter_data,
     ):
-        qkv = self.query_key_value(hidden_states)
+        qkv = self.query_key_value(hidden_states, adapter_data)
         query, kv = qkv.split(
             [
                 self.head_size * self.num_heads,
@@ -219,45 +210,42 @@ class MistralAttention(torch.nn.Module):
         else:
             kv_to_cache = kv
 
-        paged_attention.reshape_and_cache(
+        reshape_and_cache(
             kv_to_cache[:, 0], kv_to_cache[:, 1], kv_cache[0], kv_cache[1], slots
         )
 
-        # output tensor
-        attn_output = torch.empty_like(query)
-
         # Prefill
         if cu_seqlen_prefill is not None:
             # flash attention
-            flash_attn.attention(
+            attn_output = attention(
                 query,
-                torch.select(kv, dim=1, index=0),
-                torch.select(kv, dim=1, index=1),
-                attn_output,
-                cu_seqlen_prefill,
-                max_s,
+                kv_cache[0] if PREFILL_IN_KV_CACHE else kv_to_cache[:, 0],
+                kv_cache[1] if PREFILL_IN_KV_CACHE else kv_to_cache[:, 1],
+                seqlen,
+                block_tables,
                 self.softmax_scale,
                 window_size_left=self.max_past,
             )
         # Decode
         else:
-            paged_attention.attention(
-                attn_output,
+            attn_output = paged_attention(
                 query,
                 kv_cache[0],
                 kv_cache[1],
                 self.kv_head_mapping,
                 self.softmax_scale,
                 block_tables,
-                input_lengths,
+                seqlen,
                 max_s,
             )
 
-        return self.o_proj(attn_output.view(-1, self.num_heads * self.head_size))
+        return self.o_proj(
+            attn_output.view(-1, self.num_heads * self.head_size), adapter_data
+        )
 
 
 class MistralMLP(nn.Module):
-    def __init__(self, prefix, config, weights):
+    def __init__(self, prefix: str, config, weights, layer_id):
         super().__init__()
         self.hidden_act = config.hidden_act
         self.act = (
@@ -273,19 +261,37 @@ class MistralMLP(nn.Module):
             )
         )
         # Fuse gate and up proj
-        self.gate_up_proj = TensorParallelColumnLinear.load_multi(
+        gate_up_proj = TensorParallelColumnLinear.load_multi(
             config,
             prefixes=[f"{prefix}.gate_proj", f"{prefix}.up_proj"],
             weights=weights,
             dim=0,
             bias=False,
         )
-        self.down_proj = TensorParallelRowLinear.load(
+        self.gate_up_proj = TensorParallelMultiAdapterLinear.load(
+            gate_up_proj,
+            layer_id,
+            ["gate_proj", "up_proj"],
+            sizes=[
+                config.intermediate_size,
+                config.intermediate_size,
+            ],
+            process_group=weights.process_group,
+        )
+
+        down_proj = TensorParallelRowLinear.load(
             config,
             prefix=f"{prefix}.down_proj",
             weights=weights,
             bias=False,
         )
+
+        self.down_proj = TensorParallelAdapterRowLinear.load(
+            down_proj,
+            layer_id,
+            "down_proj",
+            process_group=weights.process_group,
+        )
         self.intermediate_size = (
             config.intermediate_size // weights.process_group.size()
         )
@@ -293,10 +299,11 @@ class MistralMLP(nn.Module):
         # TODO: This is a hotfix to be removed & properly refactored.
         self.quantize = config.quantize
 
-    def forward(self, hidden_states):
+    def forward(self, hidden_states, adapter_data):
         if (
             SYSTEM == "rocm"
             and self.hidden_act == "silu"
+            and hidden_states.dtype == torch.float16
             and hidden_states.shape[0] == 1
             and not self.quantize
         ):
@@ -306,21 +313,30 @@ class MistralMLP(nn.Module):
                 dtype=hidden_states.dtype,
                 device="cuda",
             )
-            _custom_C.LLMM_Silu(self.gate_up_proj.linear.weight, hidden_states, out, 8)
-            return self.down_proj(out)
+            _custom_C.LLMM_Silu(
+                self.gate_up_proj.base_layer.linear.weight, hidden_states, out, 8
+            )
+            return self.down_proj(out, adapter_data)
         else:
-            gate_up_states = self.gate_up_proj(hidden_states)
+            gate_up_states = self.gate_up_proj(hidden_states, adapter_data)
             gate_up_states = gate_up_states.view(-1, 2, self.intermediate_size)
-            return self.down_proj(self.act(gate_up_states[:, 0]) * gate_up_states[:, 1])
+            return self.down_proj(
+                self.act(gate_up_states[:, 0]) * gate_up_states[:, 1], adapter_data
+            )
 
 
 class MistralLayer(nn.Module):
-    def __init__(self, prefix, config, weights):
+    def __init__(self, prefix: str, config, weights, layer_id):
         super().__init__()
         self.self_attn = MistralAttention(
-            prefix=f"{prefix}.self_attn", config=config, weights=weights
+            prefix=f"{prefix}.self_attn",
+            config=config,
+            weights=weights,
+            layer_id=layer_id,
+        )
+        self.mlp = MistralMLP(
+            prefix=f"{prefix}.mlp", config=config, weights=weights, layer_id=layer_id
         )
-        self.mlp = MistralMLP(prefix=f"{prefix}.mlp", config=config, weights=weights)
 
         self.input_layernorm = FastRMSNorm.load(
             prefix=f"{prefix}.input_layernorm", weights=weights, eps=config.rms_norm_eps
@@ -341,9 +357,10 @@ class MistralLayer(nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
         prefill_cache_indices,
+        adapter_data,
     ):
         normed_hidden_states, res = self.input_layernorm(hidden_states, residual)
 
@@ -356,9 +373,10 @@ class MistralLayer(nn.Module):
             kv_cache,
             block_tables,
             slots,
-            input_lengths,
+            seqlen,
             max_s,
             prefill_cache_indices,
+            adapter_data,
         )
 
         # faster post attention rms norm
@@ -366,13 +384,13 @@ class MistralLayer(nn.Module):
             attn_output, res
         )
 
-        mlp_output = self.mlp(normed_attn_res_output)
+        mlp_output = self.mlp(normed_attn_res_output, adapter_data)
 
         return mlp_output, attn_res
 
 
 class MistralModel(torch.nn.Module):
-    def __init__(self, prefix, config, weights):
+    def __init__(self, prefix: str, config, weights):
         super().__init__()
 
         process_group = weights.process_group
@@ -384,6 +402,7 @@ class MistralModel(torch.nn.Module):
                     prefix=f"{prefix}.layers.{layer_id}",
                     config=config,
                     weights=weights,
+                    layer_id=layer_id,
                 )
                 for layer_id in range(config.num_hidden_layers)
             ]
@@ -406,10 +425,11 @@ class MistralModel(torch.nn.Module):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
         true_max_s: int,
         prefill_cache_indices: Optional[torch.Tensor],
+        adapter_data: Optional[torch.Tensor] = None,
     ):
         hidden_states = inputs_embeds
         # Get rotary cos and sin for this forward
@@ -429,9 +449,10 @@ class MistralModel(torch.nn.Module):
                 kv_cache[i],
                 block_tables,
                 slots,
-                input_lengths,
+                seqlen,
                 max_s,
                 prefill_cache_indices,
+                adapter_data,
             )
 
         hidden_states, _ = self.norm(hidden_states, residual)
@@ -439,7 +460,7 @@ class MistralModel(torch.nn.Module):
 
 
 class FlashMistralForCausalLM(torch.nn.Module):
-    def __init__(self, prefix, config, weights, name=None):
+    def __init__(self, prefix: str, config, weights, name=None):
         if name is None:
             name = "model"
         super().__init__()
@@ -479,10 +500,11 @@ class FlashMistralForCausalLM(torch.nn.Module):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
         prefill_cache_indices: Optional[torch.Tensor],
         lm_head_indices: Optional[torch.Tensor] = None,
+        adapter_data: Optional[torch.Tensor] = None,
     ) -> torch.Tensor:
         true_max_s = max_s
         if prefill_cache_indices is not None:
@@ -491,7 +513,7 @@ class FlashMistralForCausalLM(torch.nn.Module):
         elif self.max_past is not None:
             # Clamp in decode mode as paged attention requires clamped values whereas the flash attention
             # kernel requires the true values
-            input_lengths = torch.clamp(input_lengths, max=self.max_past_tensor)
+            seqlen = seqlen.clamp(max=self.max_past_tensor)
 
         inputs_embeds = self.embed_tokens(input_ids)
         hidden_states = self.model(
@@ -501,10 +523,11 @@ class FlashMistralForCausalLM(torch.nn.Module):
             kv_cache,
             block_tables,
             slots,
-            input_lengths,
+            seqlen,
             max_s,
             true_max_s,
             prefill_cache_indices,
+            adapter_data,
         )
         if lm_head_indices is not None:
             hidden_states = hidden_states[lm_head_indices]
diff --git a/server/text_generation_server/models/custom_modeling/flash_mixtral_modeling.py b/server/text_generation_server/models/custom_modeling/flash_mixtral_modeling.py
index be2d6c45..5836d30a 100644
--- a/server/text_generation_server/models/custom_modeling/flash_mixtral_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/flash_mixtral_modeling.py
@@ -18,36 +18,32 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from typing import List, Optional, Tuple, Type
+
 import torch
 import torch.distributed
-
-import numpy as np
-
 from torch import nn
-from text_generation_server.utils.import_utils import SYSTEM
-
-if SYSTEM != "xpu":
-    from vllm.model_executor.layers.fused_moe import fused_moe
-from transformers.activations import ACT2FN
 from transformers.configuration_utils import PretrainedConfig
-from typing import Optional, List, Tuple
-from loguru import logger
 
-from text_generation_server.utils import paged_attention, flash_attn
 from text_generation_server.layers import (
     FastLinear,
-    TensorParallelRowLinear,
+    SpeculativeHead,
     TensorParallelColumnLinear,
     TensorParallelEmbedding,
-    SpeculativeHead,
+    TensorParallelRowLinear,
     get_linear,
 )
-from text_generation_server.layers.layernorm import (
-    FastRMSNorm,
-)
-from text_generation_server.layers.rotary import (
-    PositionRotaryEmbedding,
+from text_generation_server.layers.attention import (
+    Seqlen,
+    attention,
+    paged_attention,
+    reshape_and_cache,
 )
+from text_generation_server.layers.attention import PREFILL_IN_KV_CACHE
+from text_generation_server.layers.layernorm import FastRMSNorm
+from text_generation_server.layers.moe import DenseMoELayer, MoELayer, SparseMoELayer
+from text_generation_server.layers.rotary import PositionRotaryEmbedding
+from text_generation_server.utils.weights import UnquantizedWeight
 
 
 class MixtralConfig(PretrainedConfig):
@@ -112,7 +108,7 @@ def promote_scalar(x: torch.Tensor) -> torch.Tensor:
     return x.view(1) if len(x.size()) == 0 else x
 
 
-def load_attention(config, prefix, weights):
+def load_attention(config, prefix: str, weights):
     if config.num_attention_heads != config.num_key_value_heads:
         return _load_gqa(config, prefix, weights)
     else:
@@ -131,27 +127,24 @@ def _load_gqa(config, prefix: str, weights):
 
     weight = weights.get_multi_weights_col(
         prefixes=[f"{prefix}.q_proj", f"{prefix}.k_proj", f"{prefix}.v_proj"],
-        quantize=config.quantize,
         dim=0,
     )
 
-    if config.quantize not in ["gptq", "awq"]:
-        weight = weight.to(dtype=weights.dtype).to(device=weights.device)
+    if isinstance(weight, UnquantizedWeight):
+        weight.weight = weight.weight.to(dtype=weights.dtype).to(device=weights.device)
 
         head_size = config.hidden_size // config.num_attention_heads
         num_heads = config.num_attention_heads // weights.process_group.size()
         num_key_value_heads = config.num_key_value_heads // weights.process_group.size()
-        assert list(weight.shape) == [
+        assert list(weight.weight.shape) == [
             (num_heads + 2 * num_key_value_heads) * head_size,
             config.hidden_size,
-        ], f"{list(weight.shape)} != {[(num_heads + 2 * config.num_key_value_heads) * head_size, config.hidden_size]}"
+        ], f"{list(weight.weight.shape)} != {[(num_heads + 2 * config.num_key_value_heads) * head_size, config.hidden_size]}"
 
-    return TensorParallelColumnLinear(
-        get_linear(weight, bias=None, quantize=config.quantize)
-    )
+    return TensorParallelColumnLinear(get_linear(weight, bias=None))
 
 
-def _load_experts(config, prefix, mat, weights):
+def _load_experts(config, prefix: str, mat, weights):
     if config.quantize is not None:
         raise NotImplementedError("Mixtral does not support weight quantization yet.")
 
@@ -243,7 +236,7 @@ class MixtralAttention(torch.nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
         prefill_cache_indices,
     ):
@@ -265,37 +258,32 @@ class MixtralAttention(torch.nn.Module):
         else:
             kv_to_cache = kv
 
-        paged_attention.reshape_and_cache(
+        reshape_and_cache(
             kv_to_cache[:, 0], kv_to_cache[:, 1], kv_cache[0], kv_cache[1], slots
         )
 
-        # output tensor
-        attn_output = torch.empty_like(query)
-
         # Prefill
         if cu_seqlen_prefill is not None:
             # flash attention
-            flash_attn.attention(
+            attn_output = attention(
                 query,
-                torch.select(kv, dim=1, index=0),
-                torch.select(kv, dim=1, index=1),
-                attn_output,
-                cu_seqlen_prefill,
-                max_s,
+                kv_cache[0] if PREFILL_IN_KV_CACHE else kv_to_cache[:, 0],
+                kv_cache[1] if PREFILL_IN_KV_CACHE else kv_to_cache[:, 1],
+                seqlen,
+                block_tables,
                 self.softmax_scale,
                 window_size_left=self.max_past,
             )
         # Decode
         else:
-            paged_attention.attention(
-                attn_output,
+            attn_output = paged_attention(
                 query,
                 kv_cache[0],
                 kv_cache[1],
                 self.kv_head_mapping,
                 self.softmax_scale,
                 block_tables,
-                input_lengths,
+                seqlen,
                 max_s,
             )
 
@@ -320,59 +308,35 @@ def round_up(x: torch.Tensor, value: int):
     return torch.div(x + (value - 1), value, rounding_mode="trunc") * value
 
 
-class BlockSparseMoE(nn.Module):
-    def __init__(self, prefix, config: MixtralConfig, weights):
+class MixtralMoE(nn.Module):
+    def __init__(
+        self, prefix, config: MixtralConfig, moe_layer_cls: Type[MoELayer], weights
+    ):
         super().__init__()
-        self.hidden_dim = config.hidden_size
-        self.ffn_dim = config.intermediate_size // weights.process_group.size()
-        self.num_experts = config.num_local_experts
-        self.top_k = config.num_experts_per_tok
-
-        act = config.hidden_act
-        if "gelu" in act:
-            self.act = lambda x: torch.nn.functional.gelu(
-                x,
-                approximate=(
-                    "tanh" if act in ["gelu_fast", "gelu_pytorch_tanh"] else "none"
-                ),
-            )
-        elif "silu" in act:
-            self.act = torch.nn.functional.silu
-        else:
-            self.act = ACT2FN[act]
 
         # gating
         self.gate = FastLinear.load(config, f"{prefix}.gate", weights, bias=False)
 
-        # merged expert weights, all of size  (n_experts * ffn_dim, hidden_dim)
-        w1 = _load_experts(config, f"{prefix}.experts", "w1", weights).view(
-            self.num_experts, self.ffn_dim, self.hidden_dim
-        )
-        w3 = _load_experts(config, f"{prefix}.experts", "w3", weights).view(
-            self.num_experts, self.ffn_dim, self.hidden_dim
-        )
-        self.w13 = torch.cat([w1, w3], dim=1)
-        self.w2 = (
-            _load_experts(config, f"{prefix}.experts", "w2", weights)
-            .view(self.num_experts, self.ffn_dim, self.hidden_dim)
-            .transpose(1, 2)
-            .contiguous()
+        self.moe = moe_layer_cls(
+            n_expert_group=None,
+            n_experts=config.num_local_experts,
+            prefix=f"{prefix}.experts",
+            renormalize=True,
+            topk=config.num_experts_per_tok,
+            topk_group=None,
+            weights=weights,
+            gate_proj_name="w1",
+            up_proj_name="w3",
+            down_proj_name="w2",
         )
+        assert isinstance(self.moe, MoELayer)
 
         self.process_group = weights.process_group
 
     def forward(self, x: torch.Tensor) -> torch.Tensor:
         # router_logits: (num_tokens, n_experts)
         router_logits = self.gate(x)
-        out = fused_moe(
-            x,
-            self.w13,
-            self.w2,
-            router_logits,
-            self.top_k,
-            renormalize=True,
-            inplace=True,
-        )
+        out = self.moe(x, gating_output=router_logits)
 
         # Reduce sum
         if self.process_group.size() > 1:
@@ -381,97 +345,8 @@ class BlockSparseMoE(nn.Module):
         return out.view(*x.shape)
 
 
-class DenseMoE(nn.Module):
-    def __init__(self, prefix, config: MixtralConfig, weights):
-        super().__init__()
-        self.hidden_dim = config.hidden_size
-        self.ffn_dim = config.intermediate_size // weights.process_group.size()
-        self.num_experts = config.num_local_experts
-        self.top_k = config.num_experts_per_tok
-
-        act = config.hidden_act
-        if "gelu" in act:
-            self.act = lambda x: torch.nn.functional.gelu(
-                x,
-                approximate=(
-                    "tanh" if act in ["gelu_fast", "gelu_pytorch_tanh"] else "none"
-                ),
-            )
-        elif "silu" in act:
-            self.act = torch.nn.functional.silu
-        else:
-            self.act = ACT2FN[act]
-
-        # gating
-        self.gate = FastLinear.load(config, f"{prefix}.gate", weights, bias=False)
-
-        self.w1 = [
-            TensorParallelColumnLinear.load(
-                config, prefix=f"{prefix}.experts.{i}.w1", weights=weights, bias=False
-            )
-            for i in range(self.num_experts)
-        ]
-        self.w3 = [
-            TensorParallelColumnLinear.load(
-                config, prefix=f"{prefix}.experts.{i}.w3", weights=weights, bias=False
-            )
-            for i in range(self.num_experts)
-        ]
-        self.w2 = [
-            TensorParallelRowLinear.load(
-                config, prefix=f"{prefix}.experts.{i}.w2", weights=weights, bias=False
-            )
-            for i in range(self.num_experts)
-        ]
-
-        self.process_group = weights.process_group
-
-    def forward(self, x: torch.Tensor) -> torch.Tensor:
-        """
-        x: (sequence_length, model_dim)
-        gate_logits: (sequence_length, n_experts)
-        """
-        # optional reshape
-        input_shape = x.shape
-        x = x.view(-1, input_shape[-1])
-
-        # gate_logits: (sequence_length, n_experts)
-        gate_logits = self.gate(x)
-        # all_probs: (sequence_length, n_experts) and upcast for softmax
-        all_probs = torch.nn.functional.softmax(gate_logits, dim=1, dtype=torch.float)
-
-        if self.top_k < self.num_experts:
-            _, not_selected_experts = torch.topk(
-                all_probs,
-                self.num_experts - self.top_k,
-                largest=False,
-                sorted=False,
-                dim=1,
-            )
-            # Mask not selected experts
-            all_probs.scatter_(1, not_selected_experts, 0)
-
-        # Re-normalize
-        weights = all_probs / all_probs.sum(dim=1, keepdim=True)
-        weights = weights.to(x.dtype)
-
-        # Final output tensor
-        out = x.new_zeros(x.shape[0], self.hidden_dim)
-        for i in range(self.num_experts):
-            h = self.act(self.w1[i](x)) * self.w3[i](x)
-            h = self.w2[i](h, reduce=False)
-            # Add expert output to out with masking
-            out += h * weights[:, i].view(-1, 1)
-
-        # Reduce sum
-        if self.process_group.size() > 1:
-            torch.distributed.all_reduce(out, group=self.process_group)
-
-        return out
-
-
 class MixtralLayer(nn.Module):
-    def __init__(self, prefix, layer_id, config, weights):
+    def __init__(self, prefix: str, layer_id, config, weights):
         super().__init__()
         prefix = f"{prefix}.layers.{layer_id}"
 
@@ -479,8 +354,12 @@ class MixtralLayer(nn.Module):
             prefix=f"{prefix}.self_attn", config=config, weights=weights
         )
 
-        moe_cls = BlockSparseMoE if config.quantize is None else DenseMoE
-        self.moe = moe_cls(f"{prefix}.block_sparse_moe", config, weights)
+        moe_layer_cls = (
+            SparseMoELayer if SparseMoELayer.is_supported(weights) else DenseMoELayer
+        )
+        self.moe = MixtralMoE(
+            f"{prefix}.block_sparse_moe", config, moe_layer_cls, weights
+        )
 
         self.input_layernorm = FastRMSNorm.load(
             prefix=f"{prefix}.input_layernorm", weights=weights, eps=config.rms_norm_eps
@@ -501,7 +380,7 @@ class MixtralLayer(nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
         prefill_cache_indices,
     ):
@@ -516,7 +395,7 @@ class MixtralLayer(nn.Module):
             kv_cache,
             block_tables,
             slots,
-            input_lengths,
+            seqlen,
             max_s,
             prefill_cache_indices,
         )
@@ -532,7 +411,7 @@ class MixtralLayer(nn.Module):
 
 
 class MixtralModel(torch.nn.Module):
-    def __init__(self, prefix, config, weights):
+    def __init__(self, prefix: str, config, weights):
         super().__init__()
 
         self.embed_tokens = TensorParallelEmbedding(
@@ -571,7 +450,7 @@ class MixtralModel(torch.nn.Module):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
         true_max_s: int,
         prefill_cache_indices: Optional[torch.Tensor],
@@ -595,7 +474,7 @@ class MixtralModel(torch.nn.Module):
                 kv_cache[i],
                 block_tables,
                 slots,
-                input_lengths,
+                seqlen,
                 max_s,
                 prefill_cache_indices,
             )
@@ -606,7 +485,7 @@ class MixtralModel(torch.nn.Module):
 
 
 class FlashMixtralForCausalLM(torch.nn.Module):
-    def __init__(self, prefix, config, weights):
+    def __init__(self, prefix: str, config, weights):
         super().__init__()
 
         self.model = MixtralModel(prefix, config, weights)
@@ -630,10 +509,11 @@ class FlashMixtralForCausalLM(torch.nn.Module):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
         prefill_cache_indices: Optional[torch.Tensor],
         lm_head_indices: Optional[torch.Tensor] = None,
+        adapter_data: Optional[torch.Tensor] = None,
     ) -> torch.Tensor:
         true_max_s = max_s
         if prefill_cache_indices is not None:
@@ -642,7 +522,7 @@ class FlashMixtralForCausalLM(torch.nn.Module):
         elif self.max_past is not None:
             # Clamp in decode mode as paged attention requires clamped values whereas the flash attention
             # kernel requires the true values
-            input_lengths = torch.clamp(input_lengths, max=self.max_past_tensor)
+            seqlen = seqlen.clamp(max=self.max_past_tensor)
 
         hidden_states = self.model(
             input_ids,
@@ -651,7 +531,7 @@ class FlashMixtralForCausalLM(torch.nn.Module):
             kv_cache,
             block_tables,
             slots,
-            input_lengths,
+            seqlen,
             max_s,
             true_max_s,
             prefill_cache_indices,
diff --git a/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py b/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py
index d45cab2e..ad4e382f 100644
--- a/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py
@@ -24,11 +24,14 @@ import torch.distributed
 from torch import nn
 from transformers.activations import ACT2FN
 from transformers.modeling_utils import PreTrainedModel
-from transformers.models.gpt_neox import GPTNeoXConfig
+from transformers.models.gpt_neox import GPTNeoXConfig as TransformersGPTNeoXConfig
 from typing import Optional, List, Tuple
-
-from text_generation_server.utils import paged_attention, flash_attn
-from text_generation_server.utils.flash_attn import attention
+from text_generation_server.layers.attention import (
+    paged_attention,
+    attention,
+    reshape_and_cache,
+    Seqlen,
+)
 from text_generation_server.layers import (
     TensorParallelRowLinear,
     TensorParallelColumnLinear,
@@ -36,16 +39,24 @@ from text_generation_server.layers import (
     SpeculativeHead,
     get_linear,
 )
+from text_generation_server.layers.attention import PREFILL_IN_KV_CACHE
 from text_generation_server.layers.layernorm import (
     FastLayerNorm,
 )
 from text_generation_server.layers.rotary import (
     PositionRotaryEmbedding,
 )
+from text_generation_server.utils.weights import UnquantizedWeight
+
+
+class GPTNeoXConfig(TransformersGPTNeoXConfig):
+    attribute_map = {
+        "num_key_value_heads": "num_attention_heads",
+    }
 
 
 def load_row(config, prefix: str, weights, bias: bool):
-    weight = weights.get_multi_weights_row(prefix, quantize=config.quantize)
+    weight = weights.get_weights_row(prefix)
 
     if bias and weights.process_group.rank() == 0:
         # Rank is only on the first rank process
@@ -53,7 +64,7 @@ def load_row(config, prefix: str, weights, bias: bool):
     else:
         bias = None
 
-    linear = get_linear(weight, bias, config.quantize)
+    linear = get_linear(weight, bias)
     if config.use_parallel_residual:
         return linear
     else:
@@ -61,11 +72,11 @@ def load_row(config, prefix: str, weights, bias: bool):
 
 
 def load_qkv(config, prefix: str, weights, num_heads, head_size, hidden_size):
-    weight = weights.get_multi_weights_col([prefix], quantize=config.quantize, dim=0)
-    if isinstance(weight, torch.Tensor):
+    weight = weights.get_multi_weights_col([prefix], dim=0)
+    if isinstance(weight, UnquantizedWeight):
         # Only on non quantized versions
-        weight = (
-            weight.view(
+        weight.weight = (
+            weight.weight.view(
                 num_heads,
                 3,
                 head_size,
@@ -78,7 +89,7 @@ def load_qkv(config, prefix: str, weights, num_heads, head_size, hidden_size):
     bias = weights.get_sharded(f"{prefix}.bias", dim=0)
     bias = bias.view(num_heads, 3, head_size).permute(1, 0, 2).reshape(-1)
 
-    linear = get_linear(weight, bias, config.quantize)
+    linear = get_linear(weight, bias)
     if config.use_parallel_residual:
         return linear
     else:
@@ -137,45 +148,46 @@ class FlashNeoxAttention(torch.nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
     ):
         qkv = self.query_key_value(hidden_states)
         qkv = qkv.view(-1, 3, self.num_heads, self.head_size)
 
+        # Compute rotary embeddings on rotary_ndims
+        query_rot = qkv[:, 0][..., : self.rotary_dim]
+        query_pass = qkv[:, 0][..., self.rotary_dim :]
+        key_rot = qkv[:, 1][..., : self.rotary_dim]
+        key_pass = qkv[:, 1][..., self.rotary_dim :]
+
         # Inplace rotary
-        self.rotary_emb(qkv[:, 0], qkv[:, 1], cos, sin)
+        self.rotary_emb(query_rot, key_rot, cos, sin)
+        qkv[:, 0] = torch.cat((query_rot, query_pass), dim=-1)
+        qkv[:, 1] = torch.cat((key_rot, key_pass), dim=-1)
 
-        paged_attention.reshape_and_cache(
-            qkv[:, 1], qkv[:, 2], kv_cache[0], kv_cache[1], slots
-        )
-
-        # output tensor
-        attn_output = torch.empty_like(qkv[:, 0])
+        reshape_and_cache(qkv[:, 1], qkv[:, 2], kv_cache[0], kv_cache[1], slots)
 
         # Prefill
         if cu_seqlen_prefill is not None:
             # flash attention
-            flash_attn.attention(
+            attn_output = attention(
                 qkv[:, 0],
-                qkv[:, 1],
-                qkv[:, 2],
-                attn_output,
-                cu_seqlen_prefill,
-                max_s,
+                kv_cache[0] if PREFILL_IN_KV_CACHE else qkv[:, 1],
+                kv_cache[1] if PREFILL_IN_KV_CACHE else qkv[:, 2],
+                seqlen,
+                block_tables,
                 self.softmax_scale,
             )
         # Decode
         else:
-            paged_attention.attention(
-                attn_output,
+            attn_output = paged_attention(
                 qkv[:, 0],
                 kv_cache[0],
                 kv_cache[1],
                 self.kv_head_mapping,
                 self.softmax_scale,
                 block_tables,
-                input_lengths,
+                seqlen,
                 max_s,
             )
 
@@ -245,7 +257,7 @@ class FlashNeoXLayer(nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
     ):
         if self.use_parallel_residual:
@@ -259,7 +271,7 @@ class FlashNeoXLayer(nn.Module):
                 kv_cache,
                 block_tables,
                 slots,
-                input_lengths,
+                seqlen,
                 max_s,
             )
 
@@ -283,7 +295,7 @@ class FlashNeoXLayer(nn.Module):
                 kv_cache,
                 block_tables,
                 slots,
-                input_lengths,
+                seqlen,
                 max_s,
             )
 
@@ -304,12 +316,12 @@ class FlashGPTNeoXPreTrainedModel(PreTrainedModel):
 
 
 class FlashGPTNeoXModel(FlashGPTNeoXPreTrainedModel):
-    def __init__(self, config, weights):
+    def __init__(self, prefix: str, config, weights):
         super().__init__(config)
         self.config = config
 
         self.embed_in = TensorParallelEmbedding(
-            prefix="gpt_neox.embed_in", weights=weights
+            prefix=f"{prefix}.embed_in", weights=weights
         )
 
         self.layers = nn.ModuleList(
@@ -319,7 +331,7 @@ class FlashGPTNeoXModel(FlashGPTNeoXPreTrainedModel):
             ]
         )
         self.final_layer_norm = FastLayerNorm.load(
-            prefix="gpt_neox.final_layer_norm",
+            prefix=f"{prefix}.final_layer_norm",
             weights=weights,
             eps=config.layer_norm_eps,
         )
@@ -337,7 +349,7 @@ class FlashGPTNeoXModel(FlashGPTNeoXPreTrainedModel):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
     ) -> torch.Tensor:
         hidden_states = self.embed_in(input_ids)
@@ -359,7 +371,7 @@ class FlashGPTNeoXModel(FlashGPTNeoXPreTrainedModel):
                 kv_cache[i],
                 block_tables,
                 slots,
-                input_lengths,
+                seqlen,
                 max_s,
             )
 
@@ -369,9 +381,15 @@ class FlashGPTNeoXModel(FlashGPTNeoXPreTrainedModel):
 
 
 class FlashGPTNeoXForCausalLM(FlashGPTNeoXPreTrainedModel):
-    def __init__(self, config, weights):
+    def __init__(self, prefix, config, weights):
         super().__init__(config)
-        self.gpt_neox = FlashGPTNeoXModel(config, weights)
+
+        if not prefix:
+            prefix = "gpt_neox"
+        else:
+            prefix = f"{prefix}.gpt_neox"
+
+        self.gpt_neox = FlashGPTNeoXModel(prefix, config, weights)
 
         self.embed_out = SpeculativeHead.load(
             config, prefix="embed_out", weights=weights
@@ -385,9 +403,11 @@ class FlashGPTNeoXForCausalLM(FlashGPTNeoXPreTrainedModel):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
+        prefill_cache_indices: Optional[torch.Tensor],
         lm_head_indices: Optional[torch.Tensor] = None,
+        adapter_data: Optional[torch.Tensor] = None,
     ) -> torch.Tensor:
         hidden_states = self.gpt_neox(
             input_ids,
@@ -396,7 +416,7 @@ class FlashGPTNeoXForCausalLM(FlashGPTNeoXPreTrainedModel):
             kv_cache,
             block_tables,
             slots,
-            input_lengths,
+            seqlen,
             max_s,
         )
         if lm_head_indices is not None:
diff --git a/server/text_generation_server/models/custom_modeling/flash_pali_gemma_modeling.py b/server/text_generation_server/models/custom_modeling/flash_pali_gemma_modeling.py
index 91c709e4..0024f2bb 100644
--- a/server/text_generation_server/models/custom_modeling/flash_pali_gemma_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/flash_pali_gemma_modeling.py
@@ -16,10 +16,10 @@
 import torch
 import torch.distributed
 from torch import nn
-from transformers.configuration_utils import PretrainedConfig
 from typing import Optional, List, Tuple
 
 from text_generation_server.layers.tensor_parallel import TensorParallelColumnLinear
+from text_generation_server.layers.attention import Seqlen
 from text_generation_server.models.custom_modeling.vlm import (
     load_text_model,
     load_vision_model,
@@ -35,6 +35,11 @@ class PaliGemmaForConditionalGeneration(nn.Module):
             config=config.vision_config,
             weights=weights,
         )
+        self.post_vision_tower_layernorm = nn.LayerNorm.load(
+            prefix="vision_tower.vision_model.post_layernorm",
+            weights=weights,
+            eps=config.vision_config.layer_norm_eps,
+        )
 
         self.multi_modal_projector = TensorParallelColumnLinear.load(
             config,
@@ -43,7 +48,7 @@ class PaliGemmaForConditionalGeneration(nn.Module):
             bias=True,
         )
 
-        self.vocab_size = config.vocab_size
+        self.vocab_size = config.text_config.vocab_size
         self.config = config
 
         text_config = config.text_config
@@ -66,7 +71,7 @@ class PaliGemmaForConditionalGeneration(nn.Module):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
         prefill_cache_indices: Optional[torch.Tensor] = None,
         lm_head_indices: Optional[torch.Tensor] = None,
@@ -74,6 +79,7 @@ class PaliGemmaForConditionalGeneration(nn.Module):
         # Unused here
         pixel_attention_mask: Optional[torch.BoolTensor] = None,
         image_sizes: Optional[torch.Tensor] = None,
+        adapter_data: Optional[torch.Tensor] = None,
     ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
         inputs_embeds = self.text_model.embed_tokens(input_ids)
         # TODO This is odd but apparently pali gemma position ids start at 1.
@@ -84,7 +90,10 @@ class PaliGemmaForConditionalGeneration(nn.Module):
         if pixel_values is not None:
             pixel_values = pixel_values.to(dtype=inputs_embeds.dtype)
             image_outputs = self.vision_tower(pixel_values)
-            image_features = self.multi_modal_projector(image_outputs.last_hidden_state)
+            last_hidden_state = self.post_vision_tower_layernorm(
+                image_outputs.last_hidden_state
+            )
+            image_features = self.multi_modal_projector(last_hidden_state)
 
             # mask where image or padding tokens
             mask = input_ids == self.config.image_token_index
@@ -99,7 +108,7 @@ class PaliGemmaForConditionalGeneration(nn.Module):
             kv_cache=kv_cache,
             block_tables=block_tables,
             slots=slots,
-            input_lengths=input_lengths,
+            seqlen=seqlen,
             max_s=max_s,
         )
 
diff --git a/server/text_generation_server/models/custom_modeling/flash_phi_modeling.py b/server/text_generation_server/models/custom_modeling/flash_phi_modeling.py
index f2efb538..2a0dc606 100644
--- a/server/text_generation_server/models/custom_modeling/flash_phi_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/flash_phi_modeling.py
@@ -6,7 +6,12 @@ from transformers.activations import ACT2FN
 from transformers.configuration_utils import PretrainedConfig
 from typing import Optional, List, Tuple
 
-from text_generation_server.utils import paged_attention, flash_attn
+from text_generation_server.layers.attention import (
+    paged_attention,
+    attention,
+    reshape_and_cache,
+    Seqlen,
+)
 from text_generation_server.layers import (
     TensorParallelRowLinear,
     TensorParallelColumnLinear,
@@ -14,6 +19,7 @@ from text_generation_server.layers import (
     SpeculativeHead,
     get_linear,
 )
+from text_generation_server.layers.attention import PREFILL_IN_KV_CACHE
 from text_generation_server.layers.layernorm import (
     FastLayerNorm,
 )
@@ -81,11 +87,10 @@ def _load_gqa(config, prefix: str, weights):
 
     weight = weights.get_multi_weights_col(
         prefixes=[f"{prefix}.q_proj", f"{prefix}.k_proj", f"{prefix}.v_proj"],
-        quantize=config.quantize,
         dim=0,
     )
 
-    if config.quantize not in ["gptq", "awq"]:
+    if config.quantize not in ["gptq", "awq", "marlin"]:
         weight = weight.to(dtype=weights.dtype).to(device=weights.device)
 
         head_size = config.hidden_size // config.num_attention_heads
@@ -97,9 +102,7 @@ def _load_gqa(config, prefix: str, weights):
         ], f"{list(weight.shape)} != {[(num_heads + 2 * config.num_key_value_heads) * head_size, config.hidden_size]}"
 
     # this is the same as llama except for Phi uses bias=True
-    return TensorParallelColumnLinear(
-        get_linear(weight, bias=True, quantize=config.quantize)
-    )
+    return TensorParallelColumnLinear(get_linear(weight, bias=True))
 
 
 class FlashPhiAttention(torch.nn.Module):
@@ -158,7 +161,7 @@ class FlashPhiAttention(torch.nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
     ):
         # Compute query, key, value and split
@@ -185,35 +188,28 @@ class FlashPhiAttention(torch.nn.Module):
         )
 
         # Reshape key and value and cache
-        paged_attention.reshape_and_cache(
-            kv[:, 0], kv[:, 1], kv_cache[0], kv_cache[1], slots
-        )
-
-        # output tensor
-        attn_output = torch.empty_like(query)
+        reshape_and_cache(kv[:, 0], kv[:, 1], kv_cache[0], kv_cache[1], slots)
 
         # Prefill
         if cu_seqlen_prefill is not None:
-            flash_attn.attention(
+            attn_output = attention(
                 query,
-                torch.select(kv, dim=1, index=0),
-                torch.select(kv, dim=1, index=1),
-                attn_output,
-                cu_seqlen_prefill,
-                max_s,
+                kv_cache[0] if PREFILL_IN_KV_CACHE else kv[:, 0],
+                kv_cache[1] if PREFILL_IN_KV_CACHE else kv[:, 1],
+                seqlen,
+                block_tables,
                 self.softmax_scale,
             )
         # Decode
         else:
-            paged_attention.attention(
-                attn_output,
+            attn_output = paged_attention(
                 query,
                 kv_cache[0],
                 kv_cache[1],
                 self.kv_head_mapping,
                 self.softmax_scale,
                 block_tables,
-                input_lengths,
+                seqlen,
                 max_s,
             )
 
@@ -236,7 +232,7 @@ class PhiMLP(nn.Module):
         )
 
         # llama weights are up_proj and down_proj and bias=False
-        self.up_proj = TensorParallelRowLinear.load(
+        self.up_proj = TensorParallelColumnLinear.load(
             config,
             prefix=f"{prefix}.fc1",
             weights=weights,
@@ -256,9 +252,9 @@ class PhiMLP(nn.Module):
 
 
 class FlashPhiLayer(nn.Module):
-    def __init__(self, layer_id, config, weights):
+    def __init__(self, prefix: str, layer_id, config, weights):
         super().__init__()
-        prefix = f"model.layers.{layer_id}"
+        prefix = f"{prefix}.layers.{layer_id}"
         self.self_attn = FlashPhiAttention(
             prefix=f"{prefix}.self_attn", config=config, weights=weights
         )
@@ -280,7 +276,7 @@ class FlashPhiLayer(nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
     ):
         hidden_states, res = self.input_layernorm(hidden_states, residual)
@@ -293,7 +289,7 @@ class FlashPhiLayer(nn.Module):
             kv_cache,
             block_tables,
             slots,
-            input_lengths,
+            seqlen,
             max_s,
         )
 
@@ -305,18 +301,19 @@ class FlashPhiLayer(nn.Module):
 
 
 class FlashPhiModel(torch.nn.Module):
-    def __init__(self, config, weights):
+    def __init__(self, prefix: str, config, weights):
         super().__init__()
 
         process_group = weights.process_group
         self.tp_rank = process_group.rank()
         self.tp_world_size = process_group.size()
         self.embed_tokens = TensorParallelEmbedding(
-            prefix="model.embed_tokens", weights=weights
+            prefix=f"{prefix}.embed_tokens", weights=weights
         )
         self.layers = nn.ModuleList(
             [
                 FlashPhiLayer(
+                    prefix,
                     layer_id,
                     config,
                     weights,
@@ -344,7 +341,7 @@ class FlashPhiModel(torch.nn.Module):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
     ) -> torch.Tensor:
         hidden_states = self.embed_tokens(input_ids)
@@ -366,7 +363,7 @@ class FlashPhiModel(torch.nn.Module):
                 kv_cache[i],
                 block_tables,
                 slots,
-                input_lengths,
+                seqlen,
                 max_s,
             )
 
@@ -376,10 +373,15 @@ class FlashPhiModel(torch.nn.Module):
 
 
 class FlashPhiForCausalLM(torch.nn.Module):
-    def __init__(self, config, weights):
+    def __init__(self, prefix: str, config, weights):
         super().__init__()
 
-        self.model = FlashPhiModel(config, weights)
+        if not prefix:
+            prefix = "model"
+        else:
+            prefix = f"{prefix}.model"
+
+        self.model = FlashPhiModel(prefix, config, weights)
         self.lm_head = SpeculativeHead.load(
             config,
             prefix="lm_head",
@@ -394,9 +396,11 @@ class FlashPhiForCausalLM(torch.nn.Module):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
+        prefill_cache_indices: Optional[torch.Tensor],
         lm_head_indices: Optional[torch.Tensor] = None,
+        adapter_data: Optional[torch.Tensor] = None,
     ) -> torch.Tensor:
         hidden_states = self.model(
             input_ids,
@@ -405,7 +409,7 @@ class FlashPhiForCausalLM(torch.nn.Module):
             kv_cache,
             block_tables,
             slots,
-            input_lengths,
+            seqlen,
             max_s,
         )
         if lm_head_indices is not None:
diff --git a/server/text_generation_server/models/custom_modeling/flash_phi_moe_modeling.py b/server/text_generation_server/models/custom_modeling/flash_phi_moe_modeling.py
new file mode 100644
index 00000000..bb585cc4
--- /dev/null
+++ b/server/text_generation_server/models/custom_modeling/flash_phi_moe_modeling.py
@@ -0,0 +1,254 @@
+# coding=utf-8
+# Copyright 2024 Microsoft and the HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""PyTorch Phi-MoE model."""
+
+from transformers.configuration_utils import PretrainedConfig
+from transformers.utils import logging
+
+
+logger = logging.get_logger(__name__)
+
+
+PHIMOE_PRETRAINED_CONFIG_ARCHIVE_MAP = {
+    "microsoft/Phi-3.5-MoE-instruct": "https://huggingface.co/microsoft/Phi-3.5-MoE-instruct/resolve/main/config.json",
+}
+
+
+class PhiMoEConfig(PretrainedConfig):
+    r"""
+    This is the configuration class to store the configuration of a [`PhiMoEModel`]. It is used to instantiate a Phi-MoE
+    model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
+    defaults will yield a similar configuration to that of the
+    [microsoft/Phi-3.5-MoE-instruct](https://huggingface.co/microsoft/Phi-3.5-MoE-instruct).
+
+    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+    documentation from [`PretrainedConfig`] for more information.
+
+
+    Args:
+        vocab_size (`int`, *optional*, defaults to 32064):
+            Vocabulary size of the PhiMoE model. Defines the number of different tokens that can be represented by the
+            `inputs_ids` passed when calling [`PhiMoEModel`]
+        hidden_size (`int`, *optional*, defaults to 4096):
+            Dimension of the hidden representations.
+        intermediate_size (`int`, *optional*, defaults to 6400):
+            Dimension of the MLP representations.
+        num_hidden_layers (`int`, *optional*, defaults to 32):
+            Number of hidden layers in the Transformer encoder.
+        num_attention_heads (`int`, *optional*, defaults to 32):
+            Number of attention heads for each attention layer in the Transformer encoder.
+        num_key_value_heads (`int`, *optional*, defaults to 8):
+            This is the number of key_value heads that should be used to implement Grouped Query Attention. If
+            `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
+            `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When
+            converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
+            by meanpooling all the original heads within that group. For more details checkout [this
+            paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to `8`.
+        hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
+            The non-linear activation function (function or string) in the decoder.
+        max_position_embeddings (`int`, *optional*, defaults to `4096*32`):
+            The maximum sequence length that this model might ever be used with. Mixtral's sliding window attention
+            allows sequence of up to 4096*32 tokens.
+        initializer_range (`float`, *optional*, defaults to 0.02):
+            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+        rms_norm_eps (`float`, *optional*, defaults to 1e-05):
+            The epsilon used by the rms normalization layers.
+        use_cache (`bool`, *optional*, defaults to `True`):
+            Whether or not the model should return the last key/values attentions (not used by all models). Only
+            relevant if `config.is_decoder=True`.
+        pad_token_id (`int`, *optional*):
+            The id of the padding token.
+        bos_token_id (`int`, *optional*, defaults to 1):
+            The id of the "beginning-of-sequence" token.
+        eos_token_id (`int`, *optional*, defaults to 2):
+            The id of the "end-of-sequence" token.
+        tie_word_embeddings (`bool`, *optional*, defaults to `False`):
+            Whether the model's input and output word embeddings should be tied.
+        rope_theta (`float`, *optional*, defaults to 10000.0):
+            The base period of the RoPE embeddings.
+        rope_scaling (`dict`, *optional*):
+            The scaling strategy for the RoPE embeddings. If `None`, no scaling is applied. If a dictionary, it must
+            contain the following keys: `type`, `short_factor`, `long_factor`, `short_mscale`, `long_mscale` and
+            `original_max_position_embeddings`. The `type` must be `longrope`, the `short_mscale` and `long_scale` must
+            be numbers, the `short_factor` and `long_factor` must be lists of numbers with the same length as half of
+            the attention head size and the `original_max_position_embeddings` must be an integer.
+        sliding_window (`int`, *optional*):
+            Sliding window attention window size. If not specified, will default to `262144`.
+        attention_dropout (`float`, *optional*, defaults to 0.0):
+            The dropout ratio for the attention probabilities.
+        num_experts_per_tok (`int`, *optional*, defaults to 2):
+            The number of experts to root per-token, can be also interpreted as the `top-p` routing
+            parameter
+        num_local_experts (`int`, *optional*, defaults to 16):
+            Number of experts per Sparse MLP layer.
+        output_router_logits (`bool`, *optional*, defaults to `False`):
+            Whether or not the router logits should be returned by the model. Enabeling this will also
+            allow the model to output the auxiliary loss. See [here]() for more details
+        router_aux_loss_coef (`float`, *optional*, defaults to 0.0):
+            The aux loss factor for the total loss.
+        router_jitter_noise (`float`, *optional*, defaults to 0.01):
+            Amount of noise to add to the router.
+
+    ```python
+    >>> from transformers import PhiMoEModel, PhiMoEConfig
+
+    >>> # Initializing a Phi-3 style configuration
+    >>> configuration = PhiMoEConfig.from_pretrained("microsoft/Phi-3.5-MoE-instruct")
+
+    >>> # Initializing a model from the configuration
+    >>> model = PhiMoEModel(configuration)
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+    ```"""
+
+    model_type = "phimoe"
+    keys_to_ignore_at_inference = ["past_key_values"]
+
+    def __init__(
+        self,
+        vocab_size=32064,
+        hidden_size=4096,
+        intermediate_size=6400,
+        num_hidden_layers=32,
+        num_attention_heads=32,
+        num_key_value_heads=8,
+        hidden_act="silu",
+        max_position_embeddings=4096 * 32,
+        initializer_range=0.02,
+        rms_norm_eps=1e-5,
+        use_cache=True,
+        pad_token_id=None,
+        bos_token_id=1,
+        eos_token_id=2,
+        tie_word_embeddings=False,
+        rope_theta=1e6,
+        rope_scaling=None,
+        sliding_window=None,
+        attention_dropout=0.0,
+        num_experts_per_tok=2,
+        num_local_experts=16,
+        output_router_logits=False,
+        router_aux_loss_coef=0.001,
+        router_jitter_noise=0.01,
+        input_jitter_noise=0.0,
+        attention_bias=False,
+        lm_head_bias=False,
+        **kwargs,
+    ):
+        self.vocab_size = vocab_size
+        self.max_position_embeddings = max_position_embeddings
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.sliding_window = sliding_window
+        self.attention_bias = attention_bias
+        self.lm_head_bias = lm_head_bias
+        # for backward compatibility
+        if num_key_value_heads is None:
+            num_key_value_heads = num_attention_heads
+
+        self.num_key_value_heads = num_key_value_heads
+        self.hidden_act = hidden_act
+        self.initializer_range = initializer_range
+        self.rms_norm_eps = rms_norm_eps
+        self.use_cache = use_cache
+        self.rope_theta = rope_theta
+        self.attention_dropout = attention_dropout
+
+        self.num_experts_per_tok = num_experts_per_tok
+        self.num_local_experts = num_local_experts
+        self.output_router_logits = output_router_logits
+        self.router_aux_loss_coef = router_aux_loss_coef
+        self.router_jitter_noise = router_jitter_noise
+        self.input_jitter_noise = input_jitter_noise
+
+        self.rope_scaling = rope_scaling
+        self._rope_scaling_validation()
+
+        super().__init__(
+            pad_token_id=pad_token_id,
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            tie_word_embeddings=tie_word_embeddings,
+            **kwargs,
+        )
+
+    def _rope_scaling_validation(self):
+        """
+        Validate the `rope_scaling` configuration.
+        """
+        if self.rope_scaling is None:
+            return
+
+        if not isinstance(self.rope_scaling, dict) or len(self.rope_scaling) != 6:
+            raise ValueError(
+                "`rope_scaling` must be a dictionary with three fields, `type`, `short_factor`, `long_factor`, "
+                f"`short_mscale`, `long_mscale` and `original_max_position_embeddings`, got {self.rope_scaling}"
+            )
+        rope_scaling_type = self.rope_scaling.get("type", None)
+        rope_scaling_short_factor = self.rope_scaling.get("short_factor", None)
+        rope_scaling_long_factor = self.rope_scaling.get("long_factor", None)
+        rope_scaling_short_mscale = self.rope_scaling.get("short_mscale", None)
+        rope_scaling_long_mscale = self.rope_scaling.get("long_mscale", None)
+        original_max_position_embeddings = self.rope_scaling.get(
+            "original_max_position_embeddings", None
+        )
+        if rope_scaling_type is None or rope_scaling_type not in ["longrope"]:
+            raise ValueError(
+                f"`rope_scaling`'s type field must be one of ['longrope'], got {rope_scaling_type}"
+            )
+        if not (
+            isinstance(rope_scaling_short_factor, list)
+            and all(isinstance(x, (int, float)) for x in rope_scaling_short_factor)
+        ):
+            raise ValueError(
+                f"`rope_scaling`'s short_factor field must be a list of numbers, got {rope_scaling_short_factor}"
+            )
+        if (
+            not len(rope_scaling_short_factor)
+            == self.hidden_size // self.num_attention_heads // 2
+        ):
+            raise ValueError(
+                f"`rope_scaling`'s short_factor field must have length {self.hidden_size // self.num_attention_heads // 2}, got {len(rope_scaling_short_factor)}"
+            )
+        if not (
+            isinstance(rope_scaling_long_factor, list)
+            and all(isinstance(x, (int, float)) for x in rope_scaling_long_factor)
+        ):
+            raise ValueError(
+                f"`rope_scaling`'s long_factor field must be a list of numbers, got {rope_scaling_long_factor}"
+            )
+        if (
+            not len(rope_scaling_long_factor)
+            == self.hidden_size // self.num_attention_heads // 2
+        ):
+            raise ValueError(
+                f"`rope_scaling`'s long_factor field must have length {self.hidden_size // self.num_attention_heads // 2}, got {len(rope_scaling_long_factor)}"
+            )
+        if not isinstance(rope_scaling_short_mscale, (int, float)):
+            raise ValueError(
+                f"`rope_scaling`'s short_mscale field must be a number, got {rope_scaling_short_mscale}"
+            )
+        if not isinstance(rope_scaling_long_mscale, (int, float)):
+            raise ValueError(
+                f"`rope_scaling`'s long_mscale field must be a number, got {rope_scaling_long_mscale}"
+            )
+        if not isinstance(original_max_position_embeddings, int):
+            raise ValueError(
+                f"`rope_scaling`'s original_max_position_embeddings field must be an integer, got {original_max_position_embeddings}"
+            )
diff --git a/server/text_generation_server/models/custom_modeling/flash_qwen2_modeling.py b/server/text_generation_server/models/custom_modeling/flash_qwen2_modeling.py
index 3a6d2db5..02c788d3 100644
--- a/server/text_generation_server/models/custom_modeling/flash_qwen2_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/flash_qwen2_modeling.py
@@ -5,14 +5,19 @@ from torch import nn
 from transformers.activations import ACT2FN
 from typing import Optional, List, Tuple
 
-from text_generation_server.utils import paged_attention, flash_attn
+from text_generation_server.layers.attention import (
+    paged_attention,
+    attention,
+    reshape_and_cache,
+    Seqlen,
+)
 from text_generation_server.layers import (
     TensorParallelRowLinear,
     TensorParallelColumnLinear,
     TensorParallelEmbedding,
     SpeculativeHead,
-    get_linear,
 )
+from text_generation_server.layers.attention import PREFILL_IN_KV_CACHE
 from text_generation_server.layers.rotary import PositionRotaryEmbedding
 from text_generation_server.layers.layernorm import (
     FastRMSNorm,
@@ -36,31 +41,12 @@ def _load_gqa(config, prefix: str, weights):
     assert config.hidden_size % config.num_attention_heads == 0
     assert config.num_attention_heads % weights.process_group.size() == 0
 
-    weight = weights.get_multi_weights_col(
+    return TensorParallelColumnLinear.load_multi(
+        config,
         prefixes=[f"{prefix}.q_proj", f"{prefix}.k_proj", f"{prefix}.v_proj"],
-        quantize=config.quantize,
         dim=0,
-    )
-
-    if config.quantize not in ["gptq", "awq"]:
-        weight = weight.to(dtype=weights.dtype).to(device=weights.device)
-
-        head_size = config.hidden_size // config.num_attention_heads
-        num_heads = config.num_attention_heads // weights.process_group.size()
-        num_key_value_heads = config.num_key_value_heads // weights.process_group.size()
-        assert list(weight.shape) == [
-            (num_heads + 2 * num_key_value_heads) * head_size,
-            config.hidden_size,
-        ], f"{list(weight.shape)} != {[(num_heads + 2 * config.num_key_value_heads) * head_size, config.hidden_size]}"
-
-    w = [
-        weights.get_sharded(f"{p}.bias", dim=0)
-        for p in [f"{prefix}.q_proj", f"{prefix}.k_proj", f"{prefix}.v_proj"]
-    ]
-    bias = torch.cat(w, dim=0).to(dtype=weights.dtype).to(device=weights.device)
-
-    return TensorParallelColumnLinear(
-        get_linear(weight, bias=bias, quantize=config.quantize)
+        weights=weights,
+        bias=True,
     )
 
 
@@ -120,7 +106,7 @@ class Qwen2Attention(torch.nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
         prefill_cache_indices,
     ):
@@ -142,37 +128,32 @@ class Qwen2Attention(torch.nn.Module):
         else:
             kv_to_cache = kv
 
-        paged_attention.reshape_and_cache(
+        reshape_and_cache(
             kv_to_cache[:, 0], kv_to_cache[:, 1], kv_cache[0], kv_cache[1], slots
         )
 
-        # output tensor
-        attn_output = torch.empty_like(query)
-
         # Prefill
         if cu_seqlen_prefill is not None:
             # flash attention
-            flash_attn.attention(
+            attn_output = attention(
                 query,
-                torch.select(kv, dim=1, index=0),
-                torch.select(kv, dim=1, index=1),
-                attn_output,
-                cu_seqlen_prefill,
-                max_s,
+                kv_cache[0] if PREFILL_IN_KV_CACHE else kv_to_cache[:, 0],
+                kv_cache[1] if PREFILL_IN_KV_CACHE else kv_to_cache[:, 1],
+                seqlen,
+                block_tables,
                 self.softmax_scale,
                 window_size_left=self.max_past,
             )
         # Decode
         else:
-            paged_attention.attention(
-                attn_output,
+            attn_output = paged_attention(
                 query,
                 kv_cache[0],
                 kv_cache[1],
                 self.kv_head_mapping,
                 self.softmax_scale,
                 block_tables,
-                input_lengths,
+                seqlen,
                 max_s,
             )
 
@@ -218,9 +199,9 @@ class Qwen2MLP(nn.Module):
 
 
 class Qwen2Layer(nn.Module):
-    def __init__(self, layer_id, config, weights):
+    def __init__(self, prefix, layer_id, config, weights):
         super().__init__()
-        prefix = f"model.layers.{layer_id}"
+        prefix = f"{prefix}.layers.{layer_id}"
         self.self_attn = Qwen2Attention(
             prefix=f"{prefix}.self_attn", config=config, weights=weights
         )
@@ -244,7 +225,7 @@ class Qwen2Layer(nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
         prefill_cache_indices,
     ):
@@ -259,7 +240,7 @@ class Qwen2Layer(nn.Module):
             kv_cache,
             block_tables,
             slots,
-            input_lengths,
+            seqlen,
             max_s,
             prefill_cache_indices,
         )
@@ -275,17 +256,21 @@ class Qwen2Layer(nn.Module):
 
 
 class Qwen2Model(torch.nn.Module):
-    def __init__(self, config, weights):
+    def __init__(self, prefix: str, config, weights):
         super().__init__()
+
+        prefix = f"{prefix}.model" if prefix else "model"
+
         process_group = weights.process_group
         self.tp_rank = process_group.rank()
         self.tp_world_size = process_group.size()
         self.embed_tokens = TensorParallelEmbedding(
-            prefix="model.embed_tokens", weights=weights
+            prefix=f"{prefix}.embed_tokens", weights=weights
         )
         self.layers = nn.ModuleList(
             [
                 Qwen2Layer(
+                    prefix,
                     layer_id,
                     config,
                     weights,
@@ -294,7 +279,7 @@ class Qwen2Model(torch.nn.Module):
             ]
         )
         self.norm = FastRMSNorm.load(
-            prefix="model.norm", weights=weights, eps=config.rms_norm_eps
+            prefix=f"{prefix}.norm", weights=weights, eps=config.rms_norm_eps
         )
 
         self.gradient_checkpointing = False
@@ -311,7 +296,7 @@ class Qwen2Model(torch.nn.Module):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
         true_max_s: int,
         prefill_cache_indices: Optional[torch.Tensor],
@@ -335,7 +320,7 @@ class Qwen2Model(torch.nn.Module):
                 kv_cache[i],
                 block_tables,
                 slots,
-                input_lengths,
+                seqlen,
                 max_s,
                 prefill_cache_indices,
             )
@@ -346,13 +331,19 @@ class Qwen2Model(torch.nn.Module):
 
 
 class Qwen2ForCausalLM(torch.nn.Module):
-    def __init__(self, config, weights):
+    def __init__(self, prefix: str, config, weights):
         super().__init__()
 
-        self.model = Qwen2Model(config, weights)
+        self.model = Qwen2Model(prefix, config, weights)
+
+        if config.tie_word_embeddings:
+            suffix = "model.embed_tokens"
+        else:
+            suffix = "lm_head"
+
         self.lm_head = SpeculativeHead.load(
             config,
-            prefix="lm_head",
+            prefix=f"{prefix}.{suffix}" if prefix else suffix,
             weights=weights,
         )
         self.max_past = config.sliding_window
@@ -370,10 +361,11 @@ class Qwen2ForCausalLM(torch.nn.Module):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
         prefill_cache_indices: Optional[torch.Tensor] = None,
         lm_head_indices: Optional[torch.Tensor] = None,
+        adapter_data: Optional[torch.Tensor] = None,
     ) -> torch.Tensor:
         true_max_s = max_s
         if prefill_cache_indices is not None:
@@ -382,7 +374,7 @@ class Qwen2ForCausalLM(torch.nn.Module):
         elif self.max_past is not None:
             # Clamp in decode mode as paged attention requires clamped values whereas the flash attention
             # kernel requires the true values
-            input_lengths = torch.clamp(input_lengths, max=self.max_past_tensor)
+            seqlen = seqlen.clamp(max=self.max_past_tensor)
 
         hidden_states = self.model(
             input_ids,
@@ -391,7 +383,7 @@ class Qwen2ForCausalLM(torch.nn.Module):
             kv_cache,
             block_tables,
             slots,
-            input_lengths,
+            seqlen,
             max_s,
             true_max_s,
             prefill_cache_indices,
diff --git a/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py b/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py
index fa463a19..6671d85e 100644
--- a/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py
@@ -5,7 +5,6 @@ import torch.distributed
 from torch import nn
 from transformers.configuration_utils import PretrainedConfig
 from transformers.modeling_utils import PreTrainedModel
-
 from text_generation_server.layers import (
     SpeculativeHead,
     TensorParallelColumnLinear,
@@ -13,13 +12,19 @@ from text_generation_server.layers import (
     TensorParallelRowLinear,
     get_linear,
 )
+from text_generation_server.layers.attention import PREFILL_IN_KV_CACHE
 from text_generation_server.layers.layernorm import FastLayerNorm
 from text_generation_server.layers.rotary import PositionRotaryEmbedding
-from text_generation_server.utils import flash_attn, paged_attention
+from text_generation_server.layers.attention import (
+    attention,
+    paged_attention,
+    reshape_and_cache,
+    Seqlen,
+)
 
 
 def load_row(config, prefix: str, weights, bias: bool):
-    weight = weights.get_multi_weights_row(prefix, quantize=config.quantize)
+    weight = weights.get_weights_row(prefix)
 
     if bias and weights.process_group.rank() == 0:
         # Rank is only on the first rank process
@@ -27,7 +32,7 @@ def load_row(config, prefix: str, weights, bias: bool):
     else:
         bias = None
 
-    linear = get_linear(weight, bias, config.quantize)
+    linear = get_linear(weight, bias)
     if config.parallel_attn:
         return linear
     else:
@@ -38,6 +43,7 @@ class RWConfig(PretrainedConfig):
     attribute_map = {
         "num_hidden_layers": "n_layer",
         "num_attention_heads": "n_head",
+        "num_key_value_heads": "n_head_kv",
     }
 
     def __init__(
@@ -89,7 +95,7 @@ class RWConfig(PretrainedConfig):
             else kwargs.pop("n_head", 8)
         )
         self.layer_norm_epsilon = layer_norm_epsilon
-        self.num_ln_in_parallel_attention = num_ln_in_prallel_attention
+        self.num_ln_in_parallel_attn = num_ln_in_prallel_attention
         self.initializer_range = initializer_range
         self.use_cache = use_cache
         self.hidden_dropout = hidden_dropout
@@ -123,7 +129,7 @@ class FlashRWAttention(torch.nn.Module):
     def __init__(
         self,
         config,
-        prefix,
+        prefix: str,
         weights,
     ):
         super().__init__()
@@ -176,7 +182,7 @@ class FlashRWAttention(torch.nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
     ):
         qkv = self.query_key_value(hidden_states)
@@ -194,36 +200,29 @@ class FlashRWAttention(torch.nn.Module):
         # Inplace rotary
         self.rotary_emb(query, torch.select(kv, dim=1, index=0), cos, sin)
 
-        paged_attention.reshape_and_cache(
-            kv[:, 0], kv[:, 1], kv_cache[0], kv_cache[1], slots
-        )
-
-        # output
-        attn_output = torch.empty_like(query)
+        reshape_and_cache(kv[:, 0], kv[:, 1], kv_cache[0], kv_cache[1], slots)
 
         # Prefill
         if cu_seqlen_prefill is not None:
             # flash attention
-            flash_attn.attention(
+            attn_output = attention(
                 query,
-                torch.select(kv, dim=1, index=0),
-                torch.select(kv, dim=1, index=1),
-                attn_output,
-                cu_seqlen_prefill,
-                max_s,
+                kv_cache[0] if PREFILL_IN_KV_CACHE else kv[:, 0],
+                kv_cache[1] if PREFILL_IN_KV_CACHE else kv[:, 1],
+                seqlen,
+                block_tables,
                 self.softmax_scale,
             )
         # Decode
         else:
-            paged_attention.attention(
-                attn_output,
+            attn_output = paged_attention(
                 query,
                 kv_cache[0],
                 kv_cache[1],
                 self.kv_head_mapping,
                 self.softmax_scale,
                 block_tables,
-                input_lengths,
+                seqlen,
                 max_s,
             )
 
@@ -234,7 +233,7 @@ class FlashRWLargeAttention(torch.nn.Module):
     def __init__(
         self,
         config,
-        prefix,
+        prefix: str,
         weights,
     ):
         super().__init__()
@@ -296,7 +295,7 @@ class FlashRWLargeAttention(torch.nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
     ):
         qkv = self.query_key_value(hidden_states)
@@ -313,7 +312,7 @@ class FlashRWLargeAttention(torch.nn.Module):
         # Inplace rotary
         self.rotary_emb(query, torch.select(kv, dim=2, index=0), cos, sin)
 
-        paged_attention.reshape_and_cache(
+        reshape_and_cache(
             kv[:, :, 0].contiguous(),
             kv[:, :, 1].contiguous(),
             kv_cache[0],
@@ -321,32 +320,27 @@ class FlashRWLargeAttention(torch.nn.Module):
             slots,
         )
 
-        # output
-        attn_output = torch.empty_like(query)
-
         # Prefill
         if cu_seqlen_prefill is not None:
             # flash attention
-            flash_attn.attention(
+            attn_output = attention(
                 query,
-                torch.select(kv, dim=2, index=0),
-                torch.select(kv, dim=2, index=1),
-                attn_output,
-                cu_seqlen_prefill,
-                max_s,
+                kv_cache[0] if PREFILL_IN_KV_CACHE else kv[:, :, 0].contiguous(),
+                kv_cache[1] if PREFILL_IN_KV_CACHE else kv[:, :, 1].contiguous(),
+                seqlen,
+                block_tables,
                 self.softmax_scale,
             )
         # Decode
         else:
-            paged_attention.attention(
-                attn_output,
+            attn_output = paged_attention(
                 query,
                 kv_cache[0],
                 kv_cache[1],
                 self.kv_head_mapping,
                 self.softmax_scale,
                 block_tables,
-                input_lengths,
+                seqlen,
                 max_s,
             )
 
@@ -356,7 +350,7 @@ class FlashRWLargeAttention(torch.nn.Module):
 
 
 class FlashMLP(nn.Module):
-    def __init__(self, config, prefix, weights):
+    def __init__(self, config, prefix: str, weights):
         super().__init__()
         self.act = torch.nn.functional.gelu
 
@@ -378,6 +372,7 @@ class FlashRWLayer(nn.Module):
     def __init__(
         self,
         layer_id,
+        prefix: str,
         config,
         weights,
     ):
@@ -386,10 +381,15 @@ class FlashRWLayer(nn.Module):
         parallel_attn = config.parallel_attn
         self.parallel_attn = parallel_attn
 
-        prefix = f"transformer.h.{layer_id}"
+        prefix = f"{prefix}.h.{layer_id}"
+
+        # NOTE: Falcon 180B uses the ln_attn prefix
+        ln_prefix = "input_layernorm"
+        if config.num_hidden_layers == 80:
+            ln_prefix = "ln_attn"
 
         self.input_layernorm = FastLayerNorm.load(
-            prefix=f"{prefix}.input_layernorm",
+            prefix=f"{prefix}.{ln_prefix}",
             weights=weights,
             eps=config.layer_norm_epsilon,
         )
@@ -426,7 +426,7 @@ class FlashRWLayer(nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
     ):
         if self.parallel_attn:
@@ -440,7 +440,7 @@ class FlashRWLayer(nn.Module):
                 kv_cache,
                 block_tables,
                 slots,
-                input_lengths,
+                seqlen,
                 max_s,
             )
 
@@ -462,7 +462,7 @@ class FlashRWLayer(nn.Module):
                 kv_cache,
                 block_tables,
                 slots,
-                input_lengths,
+                seqlen,
                 max_s,
             )
 
@@ -477,9 +477,15 @@ class FlashRWLayer(nn.Module):
 
 
 class FlashRWLayerNorm(nn.Module):
-    def __init__(self, config, prefix, weights):
+    def __init__(self, config, prefix: str, weights):
         super().__init__()
-        self.num_ln = config.num_ln_in_parallel_attn
+        # Falcon2 includes the number of layer norms in the config
+        # in the case no number of layer norms is provided, we default to 1
+        self.num_ln = getattr(config, "num_ln_in_parallel_attn", 1)
+
+        # Falcon 180B uses the ln_attn prefix and has 2 layer norms
+        if config.num_hidden_layers == 80:
+            self.num_ln = 2
 
         if self.num_ln == 1:
             self.input_ln = FastLayerNorm.load(
@@ -516,9 +522,9 @@ class FlashRWLayerNorm(nn.Module):
 
 
 class FlashRWLargeLayer(nn.Module):
-    def __init__(self, layer_id, config, weights):
+    def __init__(self, layer_id, prefix: str, config, weights):
         super().__init__()
-        prefix = f"transformer.h.{layer_id}"
+        prefix = f"{prefix}.h.{layer_id}"
 
         self.ln_layer = FlashRWLayerNorm(config, prefix, weights)
 
@@ -543,7 +549,7 @@ class FlashRWLargeLayer(nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
     ):
         # Layer norm.
@@ -558,7 +564,7 @@ class FlashRWLargeLayer(nn.Module):
             kv_cache,
             block_tables,
             slots,
-            input_lengths,
+            seqlen,
             max_s,
         )
 
@@ -578,18 +584,18 @@ class FlashRWPreTrainedModel(PreTrainedModel):
 
 
 class FlashRWModel(FlashRWPreTrainedModel):
-    def __init__(self, config, weights):
+    def __init__(self, prefix: str, config, weights):
         super().__init__(config)
         self.config = config
 
         self.word_embeddings = TensorParallelEmbedding(
-            prefix="transformer.word_embeddings", weights=weights
+            prefix=f"{prefix}.word_embeddings", weights=weights
         )
 
         if config.new_decoder_architecture:
             self.h = nn.ModuleList(
                 [
-                    FlashRWLargeLayer(layer_id, config, weights)
+                    FlashRWLargeLayer(layer_id, prefix, config, weights)
                     for layer_id in range(config.num_hidden_layers)
                 ]
             )
@@ -597,14 +603,14 @@ class FlashRWModel(FlashRWPreTrainedModel):
         else:
             self.h = nn.ModuleList(
                 [
-                    FlashRWLayer(layer_id, config, weights)
+                    FlashRWLayer(layer_id, prefix, config, weights)
                     for layer_id in range(config.num_hidden_layers)
                 ]
             )
             self.cache_size = self.h[0].self_attention.num_heads_kv
 
         self.ln_f = FastLayerNorm.load(
-            prefix="transformer.ln_f",
+            prefix=f"{prefix}.ln_f",
             weights=weights,
             eps=config.layer_norm_epsilon,
         )
@@ -619,7 +625,7 @@ class FlashRWModel(FlashRWPreTrainedModel):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
     ) -> torch.Tensor:
         hidden_states = self.word_embeddings(input_ids)
@@ -641,7 +647,7 @@ class FlashRWModel(FlashRWPreTrainedModel):
                 kv_cache[i],
                 block_tables,
                 slots,
-                input_lengths,
+                seqlen,
                 max_s,
             )
 
@@ -651,10 +657,15 @@ class FlashRWModel(FlashRWPreTrainedModel):
 
 
 class FlashRWForCausalLM(FlashRWPreTrainedModel):
-    def __init__(self, config, weights):
+    def __init__(self, prefix: str, config, weights):
         super().__init__(config)
 
-        self.transformer = FlashRWModel(config, weights)
+        if not prefix:
+            prefix = "transformer"
+        else:
+            prefix = f"{prefix}.transformer"
+
+        self.transformer = FlashRWModel(prefix, config, weights)
 
         self.lm_head = SpeculativeHead.load(config, prefix="lm_head", weights=weights)
 
@@ -666,9 +677,11 @@ class FlashRWForCausalLM(FlashRWPreTrainedModel):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
+        prefill_cache_indices: Optional[torch.Tensor],
         lm_head_indices: Optional[torch.Tensor] = None,
+        adapter_data: Optional[torch.Tensor] = None,
     ) -> torch.Tensor:
         hidden_states = self.transformer(
             input_ids,
@@ -677,7 +690,7 @@ class FlashRWForCausalLM(FlashRWPreTrainedModel):
             kv_cache,
             block_tables,
             slots,
-            input_lengths,
+            seqlen,
             max_s,
         )
         if lm_head_indices is not None:
diff --git a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py
index d2f6d9af..43eb9687 100644
--- a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py
@@ -5,7 +5,12 @@ from torch import nn
 from transformers.activations import ACT2FN
 from typing import Optional, List, Tuple
 
-from text_generation_server.utils import paged_attention, flash_attn
+from text_generation_server.layers.attention import (
+    paged_attention,
+    attention,
+    reshape_and_cache,
+    Seqlen,
+)
 from text_generation_server.layers import (
     TensorParallelRowLinear,
     TensorParallelColumnLinear,
@@ -13,6 +18,8 @@ from text_generation_server.layers import (
     TensorParallelEmbedding,
     get_linear,
 )
+from text_generation_server.layers.attention import PREFILL_IN_KV_CACHE
+from text_generation_server.layers.gptq import GPTQWeightsLoader
 from text_generation_server.layers.layernorm import (
     FastLayerNorm,
 )
@@ -25,6 +32,10 @@ def load_multi_mqa(
         return _load_multi_mqa_gptq(
             config, prefix, weights, bias, head_size, num_heads, hidden_size
         )
+    elif config.quantize == "marlin":
+        raise RuntimeError(
+            "santacoder models with marlin quantization are not yet supported"
+        )
     else:
         return _load_multi_mqa(
             config, prefix, weights, bias, head_size, num_heads, hidden_size
@@ -34,6 +45,8 @@ def load_multi_mqa(
 def _load_multi_mqa_gptq(
     config, prefix: str, weights, bias: bool, head_size, num_heads, hidden_size
 ):
+    from text_generation_server.layers.gptq import GPTQWeight
+
     if any("c_attn" in k for k in weights.routing.keys()) and not config.transpose:
         world_size = weights.process_group.size()
         rank = weights.process_group.rank()
@@ -71,16 +84,13 @@ def _load_multi_mqa_gptq(
         qzeros = torch.cat([q_tensor, kv_tensor], dim=1)
         qzeros = qzeros.to(device=weights.device)
 
-        (
-            bits,
-            groupsize,
-            _,
-            quant_method,
-        ) = weights._get_gptq_params()
-        if quant_method == "gptq":
+        loader = weights.weights_loader
+        assert isinstance(loader, GPTQWeightsLoader)
+        loader._get_gptq_params(weights)
+        if loader.quant_method == "gptq":
             g_idx = weights.get_tensor(f"{prefix}.c_attn.g_idx")
             g_idx = g_idx.to(device=weights.device)
-        elif quant_method == "awq":
+        elif loader.quant_method == "awq":
             g_idx = None
             from text_generation_server.layers.awq.conversion_utils import (
                 fast_awq_to_gptq,
@@ -90,8 +100,16 @@ def _load_multi_mqa_gptq(
 
         from text_generation_server.layers.gptq import HAS_EXLLAMA
 
-        use_exllama = HAS_EXLLAMA
-        weight = (qweight, qzeros, scales, g_idx, bits, groupsize, use_exllama)
+        weight = GPTQWeight(
+            qweight=qweight,
+            qzeros=qzeros,
+            scales=scales,
+            g_idx=g_idx,
+            bits=loader.bits,
+            groupsize=loader.groupsize,
+            use_awq_kernel=loader.quantize == "awq",
+            use_exllama=HAS_EXLLAMA,
+        )
 
         if bias:
             slice_ = weights._get_slice(f"{prefix}.c_attn.bias")
@@ -106,7 +124,7 @@ def _load_multi_mqa_gptq(
             bias = torch.cat([q_tensor, kv_tensor], dim=0)
             bias = bias.to(device=weights.device)
 
-        return TensorParallelColumnLinear(get_linear(weight, bias, config.quantize))
+        return TensorParallelColumnLinear(get_linear(weight, bias))
     else:
         raise NotImplementedError("Gptq loading with santacoder is not implemented")
 
@@ -178,29 +196,27 @@ def _load_multi_mqa(
         assert list(bias.shape) == [
             (num_heads + 2) * head_size
         ], f"{weight.shape} != {[(num_heads + 2) * head_size]}"
-    return TensorParallelColumnLinear(get_linear(weight, bias, config.quantize))
+    return TensorParallelColumnLinear(get_linear(weight, bias))
 
 
 def load_col(config, prefix: str, weights, bias: bool):
     if config.transpose:
         weight = weights.get_sharded(f"{prefix}.weight", dim=1).T
     else:
-        weight = weights.get_multi_weights_col(
-            [prefix], quantize=config.quantize, dim=0
-        )
+        weight = weights.get_multi_weights_col([prefix], dim=0)
 
     if bias:
         bias = weights.get_sharded(f"{prefix}.bias", dim=0)
     else:
         bias = None
-    return TensorParallelColumnLinear(get_linear(weight, bias, config.quantize))
+    return TensorParallelColumnLinear(get_linear(weight, bias))
 
 
 def load_row(config, prefix: str, weights, bias: bool):
     if config.transpose:
         weight = weights.get_sharded(f"{prefix}.weight", dim=0).T
     else:
-        weight = weights.get_multi_weights_row(prefix, quantize=config.quantize)
+        weight = weights.get_weights_row(prefix)
 
     if bias and weights.process_group.rank() == 0:
         # Rank is only on the first rank process
@@ -208,7 +224,7 @@ def load_row(config, prefix: str, weights, bias: bool):
     else:
         bias = None
     return TensorParallelRowLinear(
-        get_linear(weight, bias, config.quantize), process_group=weights.process_group
+        get_linear(weight, bias), process_group=weights.process_group
     )
 
 
@@ -254,7 +270,7 @@ class FlashMQAttention(torch.nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
     ):
         qkv = self.c_attn(hidden_states)
@@ -268,36 +284,31 @@ class FlashMQAttention(torch.nn.Module):
         query = query.view(-1, self.num_heads, self.head_size)
         key_value = key_value.view(-1, 2, 1, self.head_size)
 
-        paged_attention.reshape_and_cache(
+        reshape_and_cache(
             key_value[:, 0], key_value[:, 1], kv_cache[0], kv_cache[1], slots
         )
 
-        # output
-        attn_output = torch.empty_like(query)
-
         # Prefill
         if cu_seqlen_prefill is not None:
             # flash attention
-            flash_attn.attention(
+            attn_output = attention(
                 query,
-                torch.select(key_value, dim=1, index=0),
-                torch.select(key_value, dim=1, index=1),
-                attn_output,
-                cu_seqlen_prefill,
-                max_s,
+                kv_cache[0] if PREFILL_IN_KV_CACHE else key_value[:, 0],
+                kv_cache[1] if PREFILL_IN_KV_CACHE else key_value[:, 1],
+                seqlen,
+                block_tables,
                 self.softmax_scale,
             )
         # Decode
         else:
-            paged_attention.attention(
-                attn_output,
+            attn_output = paged_attention(
                 query,
                 kv_cache[0],
                 kv_cache[1],
                 self.kv_head_mapping,
                 self.softmax_scale,
                 block_tables,
-                input_lengths,
+                seqlen,
                 max_s,
             )
 
@@ -334,16 +345,16 @@ class MLP(nn.Module):
 
 
 class Block(nn.Module):
-    def __init__(self, layer_id, config, weights):
+    def __init__(self, prefix: str, layer_id, config, weights):
         super().__init__()
-        prefix = f"transformer.h.{layer_id}"
+        prefix = f"{prefix}.h.{layer_id}"
         self.ln_1 = FastLayerNorm.load(
             prefix=f"{prefix}.ln_1", weights=weights, eps=config.layer_norm_epsilon
         )
         self.ln_2 = FastLayerNorm.load(
             prefix=f"{prefix}.ln_2", weights=weights, eps=config.layer_norm_epsilon
         )
-        self.attn = FlashMQAttention(
+        self.self_attn = FlashMQAttention(
             prefix=f"{prefix}.attn",
             config=config,
             weights=weights,
@@ -362,17 +373,17 @@ class Block(nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
     ):
         hidden_states, residual = self.ln_1(hidden_states, residual)
-        hidden_states = self.attn(
+        hidden_states = self.self_attn(
             hidden_states,
             cu_seqlen_prefill,
             kv_cache,
             block_tables,
             slots,
-            input_lengths,
+            seqlen,
             max_s,
         )
 
@@ -384,25 +395,26 @@ class Block(nn.Module):
 
 
 class FlashSantacoderModel(nn.Module):
-    def __init__(self, config, weights):
+    def __init__(self, prefix: str, config, weights):
         super().__init__()
         self.config = config
 
         self.process_group = weights.process_group
         self.wte = TensorParallelEmbedding(
-            prefix="transformer.wte",
+            prefix=f"{prefix}.wte",
             weights=weights,
             reduce=False,
         )
         self.wpe = TensorParallelEmbedding(
-            prefix="transformer.wpe",
+            prefix=f"{prefix}.wpe",
             weights=weights,
             reduce=False,
         )
 
-        self.h = nn.ModuleList(
+        self.layers = nn.ModuleList(
             [
                 Block(
+                    prefix,
                     layer_id,
                     config,
                     weights,
@@ -414,8 +426,8 @@ class FlashSantacoderModel(nn.Module):
             prefix="transformer.ln_f", weights=weights, eps=config.layer_norm_epsilon
         )
 
-        self.head_size = self.h[0].attn.head_size
-        self.num_heads = self.h[0].attn.num_heads
+        self.head_size = self.layers[0].self_attn.head_size
+        self.num_heads = self.layers[0].self_attn.num_heads
 
     def forward(
         self,
@@ -425,7 +437,7 @@ class FlashSantacoderModel(nn.Module):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
     ) -> torch.Tensor:
         hidden_states = self.wte(input_ids) + self.wpe(position_ids)
@@ -434,7 +446,7 @@ class FlashSantacoderModel(nn.Module):
             torch.distributed.all_reduce(hidden_states, group=self.process_group)
 
         residual = None
-        for i, layer in enumerate(self.h):
+        for i, layer in enumerate(self.layers):
             hidden_states, residual = layer(
                 hidden_states,
                 residual,
@@ -442,7 +454,7 @@ class FlashSantacoderModel(nn.Module):
                 kv_cache[i],
                 block_tables,
                 slots,
-                input_lengths,
+                seqlen,
                 max_s,
             )
 
@@ -452,11 +464,18 @@ class FlashSantacoderModel(nn.Module):
 
 
 class FlashSantacoderForCausalLM(nn.Module):
-    def __init__(self, config, weights):
+    def __init__(self, prefix, config, weights):
         super().__init__()
-        self.transformer = FlashSantacoderModel(config, weights)
+
+        if not prefix:
+            prefix = "transformer"
+        else:
+            prefix = f"{prefix}.transformer"
+
+        config.transpose = config.architectures[0].startswith("GPT2")
+        self.model = FlashSantacoderModel(prefix, config, weights)
         self.lm_head = SpeculativeHead.load(
-            config, prefix="transformer.wte", weights=weights
+            config, prefix=f"{prefix}.wte", weights=weights
         )
 
     def forward(
@@ -467,18 +486,20 @@ class FlashSantacoderForCausalLM(nn.Module):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
+        prefill_cache_indices: Optional[torch.Tensor],
         lm_head_indices: Optional[torch.Tensor] = None,
+        adapter_data: Optional[torch.Tensor] = None,
     ) -> torch.Tensor:
-        hidden_states = self.transformer(
+        hidden_states = self.model(
             input_ids,
             position_ids,
             cu_seqlen_prefill,
             kv_cache,
             block_tables,
             slots,
-            input_lengths,
+            seqlen,
             max_s,
         )
         if lm_head_indices is not None:
diff --git a/server/text_generation_server/models/custom_modeling/flash_starcoder2_modeling.py b/server/text_generation_server/models/custom_modeling/flash_starcoder2_modeling.py
index 3e2ce4f9..4975cf22 100644
--- a/server/text_generation_server/models/custom_modeling/flash_starcoder2_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/flash_starcoder2_modeling.py
@@ -26,7 +26,12 @@ from transformers.activations import ACT2FN
 from transformers.configuration_utils import PretrainedConfig
 from typing import Optional, List, Tuple
 
-from text_generation_server.utils import paged_attention, flash_attn
+from text_generation_server.layers.attention import (
+    paged_attention,
+    attention,
+    reshape_and_cache,
+    Seqlen,
+)
 from text_generation_server.layers import (
     TensorParallelRowLinear,
     TensorParallelColumnLinear,
@@ -34,6 +39,7 @@ from text_generation_server.layers import (
     SpeculativeHead,
     get_linear,
 )
+from text_generation_server.layers.attention import PREFILL_IN_KV_CACHE
 from text_generation_server.layers.layernorm import (
     FastLayerNorm,
     FastRMSNorm,
@@ -41,6 +47,7 @@ from text_generation_server.layers.layernorm import (
 from text_generation_server.layers.rotary import (
     PositionRotaryEmbedding,
 )
+from text_generation_server.utils.weights import UnquantizedWeight
 
 
 class Starcoder2Config(PretrainedConfig):
@@ -122,20 +129,19 @@ def _load_gqa(config, prefix: str, weights):
 
     weight = weights.get_multi_weights_col(
         prefixes=[f"{prefix}.q_proj", f"{prefix}.k_proj", f"{prefix}.v_proj"],
-        quantize=config.quantize,
         dim=0,
     )
 
-    if config.quantize not in ["gptq", "awq"]:
-        weight = weight.to(dtype=weights.dtype).to(device=weights.device)
+    if isinstance(weight, UnquantizedWeight):
+        weight.weight = weight.weight.to(dtype=weights.dtype).to(device=weights.device)
 
         head_size = config.hidden_size // config.num_attention_heads
         num_heads = config.num_attention_heads // weights.process_group.size()
         num_key_value_heads = config.num_key_value_heads // weights.process_group.size()
-        assert list(weight.shape) == [
+        assert list(weight.weight.shape) == [
             (num_heads + 2 * num_key_value_heads) * head_size,
             config.hidden_size,
-        ], f"{list(weight.shape)} != {[(num_heads + 2 * config.num_key_value_heads) * head_size, config.hidden_size]}"
+        ], f"{list(weight.weight.shape)} != {[(num_heads + 2 * config.num_key_value_heads) * head_size, config.hidden_size]}"
 
     if config.use_bias:
         w = [
@@ -146,9 +152,7 @@ def _load_gqa(config, prefix: str, weights):
     else:
         bias = None
 
-    return TensorParallelColumnLinear(
-        get_linear(weight, bias=bias, quantize=config.quantize)
-    )
+    return TensorParallelColumnLinear(get_linear(weight, bias=bias))
 
 
 class Starcoder2Attention(torch.nn.Module):
@@ -207,7 +211,7 @@ class Starcoder2Attention(torch.nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
         prefill_cache_indices,
     ):
@@ -229,37 +233,32 @@ class Starcoder2Attention(torch.nn.Module):
         else:
             kv_to_cache = kv
 
-        paged_attention.reshape_and_cache(
+        reshape_and_cache(
             kv_to_cache[:, 0], kv_to_cache[:, 1], kv_cache[0], kv_cache[1], slots
         )
 
-        # output tensor
-        attn_output = torch.empty_like(query)
-
         # Prefill
         if cu_seqlen_prefill is not None:
             # flash attention
-            flash_attn.attention(
+            attn_output = attention(
                 query,
-                torch.select(kv, dim=1, index=0),
-                torch.select(kv, dim=1, index=1),
-                attn_output,
-                cu_seqlen_prefill,
-                max_s,
+                kv_cache[0] if PREFILL_IN_KV_CACHE else kv_to_cache[:, 0],
+                kv_cache[1] if PREFILL_IN_KV_CACHE else kv_to_cache[:, 1],
+                seqlen,
+                block_tables,
                 self.softmax_scale,
                 window_size_left=self.max_past,
             )
         # Decode
         else:
-            paged_attention.attention(
-                attn_output,
+            attn_output = paged_attention(
                 query,
                 kv_cache[0],
                 kv_cache[1],
                 self.kv_head_mapping,
                 self.softmax_scale,
                 block_tables,
-                input_lengths,
+                seqlen,
                 max_s,
             )
 
@@ -382,7 +381,7 @@ class Starcoder2Layer(nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
         prefill_cache_indices,
     ):
@@ -397,7 +396,7 @@ class Starcoder2Layer(nn.Module):
             kv_cache,
             block_tables,
             slots,
-            input_lengths,
+            seqlen,
             max_s,
             prefill_cache_indices,
         )
@@ -413,14 +412,14 @@ class Starcoder2Layer(nn.Module):
 
 
 class Starcoder2Model(torch.nn.Module):
-    def __init__(self, config, weights):
+    def __init__(self, prefix, config, weights):
         super().__init__()
 
         process_group = weights.process_group
         self.tp_rank = process_group.rank()
         self.tp_world_size = process_group.size()
         self.embed_tokens = TensorParallelEmbedding(
-            prefix="model.embed_tokens", weights=weights
+            prefix=f"{prefix}.embed_tokens", weights=weights
         )
         self.layers = nn.ModuleList(
             [
@@ -433,7 +432,7 @@ class Starcoder2Model(torch.nn.Module):
             ]
         )
         self.norm = STARCODER2_NORMALIZATION_CLASSES[config.norm_type].load(
-            prefix="model.norm", weights=weights, eps=config.norm_epsilon
+            prefix=f"{prefix}.norm", weights=weights, eps=config.norm_epsilon
         )
 
         self.gradient_checkpointing = False
@@ -450,7 +449,7 @@ class Starcoder2Model(torch.nn.Module):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
         true_max_s: int,
         prefill_cache_indices: Optional[torch.Tensor],
@@ -474,7 +473,7 @@ class Starcoder2Model(torch.nn.Module):
                 kv_cache[i],
                 block_tables,
                 slots,
-                input_lengths,
+                seqlen,
                 max_s,
                 prefill_cache_indices,
             )
@@ -485,10 +484,15 @@ class Starcoder2Model(torch.nn.Module):
 
 
 class FlashStarcoder2ForCausalLM(torch.nn.Module):
-    def __init__(self, config, weights):
+    def __init__(self, prefix, config, weights):
         super().__init__()
 
-        self.model = Starcoder2Model(config, weights)
+        if not prefix:
+            prefix = "model"
+        else:
+            prefix = f"{prefix}.model"
+
+        self.model = Starcoder2Model(prefix, config, weights)
         try:
             self.lm_head = SpeculativeHead.load(
                 config,
@@ -498,7 +502,7 @@ class FlashStarcoder2ForCausalLM(torch.nn.Module):
         except RuntimeError:
             self.lm_head = SpeculativeHead.load(
                 config,
-                prefix="model.embed_tokens",
+                prefix=f"{prefix}.embed_tokens",
                 weights=weights,
             )
 
@@ -517,10 +521,11 @@ class FlashStarcoder2ForCausalLM(torch.nn.Module):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
         prefill_cache_indices: Optional[torch.Tensor],
         lm_head_indices: Optional[torch.Tensor] = None,
+        adapter_data: Optional[torch.Tensor] = None,
     ) -> torch.Tensor:
         true_max_s = max_s
         if prefill_cache_indices is not None:
@@ -529,7 +534,7 @@ class FlashStarcoder2ForCausalLM(torch.nn.Module):
         elif self.max_past is not None:
             # Clamp in decode mode as paged attention requires clamped values whereas the flash attention
             # kernel requires the true values
-            input_lengths = torch.clamp(input_lengths, max=self.max_past_tensor)
+            seqlen = seqlen.clamp(max=self.max_past_tensor)
 
         hidden_states = self.model(
             input_ids,
@@ -538,7 +543,7 @@ class FlashStarcoder2ForCausalLM(torch.nn.Module):
             kv_cache,
             block_tables,
             slots,
-            input_lengths,
+            seqlen,
             max_s,
             true_max_s,
             prefill_cache_indices,
diff --git a/server/text_generation_server/models/custom_modeling/idefics2.py b/server/text_generation_server/models/custom_modeling/idefics2.py
index 51fd7c02..a829c374 100644
--- a/server/text_generation_server/models/custom_modeling/idefics2.py
+++ b/server/text_generation_server/models/custom_modeling/idefics2.py
@@ -14,7 +14,7 @@
 # limitations under the License.
 """ PyTorch Idefics2 model."""
 
-from typing import List, Optional, Tuple, Union
+from typing import List, Optional, Tuple
 
 import torch
 import torch.utils.checkpoint
@@ -22,11 +22,10 @@ from torch import nn
 import math
 
 from transformers.activations import ACT2FN
-from transformers.image_processing_utils import select_best_resolution
 from text_generation_server.models.custom_modeling.vlm import (
     load_text_model,
-    load_vision_model,
 )
+from text_generation_server.layers.attention import Seqlen
 from transformers.modeling_attn_mask_utils import _prepare_4d_attention_mask
 
 from text_generation_server.layers import (
@@ -34,6 +33,7 @@ from text_generation_server.layers import (
     TensorParallelEmbedding,
     TensorParallelRowLinear,
 )
+from text_generation_server.utils.weights import DefaultWeightsLoader, UnquantizedWeight
 
 
 def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
@@ -682,7 +682,7 @@ class Idefics2Connector(nn.Module):
 class Idefics2ForConditionalGeneration(nn.Module):
     def __init__(self, prefix, config, weights):
         super().__init__()
-        config.vision_config.quantize = config.quantize
+        config.vision_config.quantize = None
         config.vision_config.speculator = config.speculator
         config.text_config.quantize = config.quantize
         config.text_config.speculator = config.speculator
@@ -695,16 +695,24 @@ class Idefics2ForConditionalGeneration(nn.Module):
             name="text_model",
         )
         self.dtype = weights.dtype
-        self.vision_model = Idefics2VisionTransformer(
-            prefix=f"{prefix}.model.vision_model" if prefix else "model.vision_model",
-            config=vision_config,
-            weights=weights,
-        )
-        self.connector = Idefics2Connector(
-            prefix=f"{prefix}.model.connector" if prefix else "model.connector",
-            config=config,
-            weights=weights,
-        )
+
+        # The vision and connector models are not quantized.
+        with weights.use_loader(DefaultWeightsLoader(UnquantizedWeight)):
+            self.vision_model = Idefics2VisionTransformer(
+                prefix=(
+                    f"{prefix}.model.vision_model" if prefix else "model.vision_model"
+                ),
+                config=vision_config,
+                weights=weights,
+            )
+
+            config.quantize = None
+            self.connector = Idefics2Connector(
+                prefix=f"{prefix}.model.connector" if prefix else "model.connector",
+                config=config,
+                weights=weights,
+            )
+
         self.config = config
         self.image_seq_len = config.perceiver_config.resampler_n_latents
         self.image_token_id = config.image_token_id
@@ -733,7 +741,7 @@ class Idefics2ForConditionalGeneration(nn.Module):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
         prefill_cache_indices: Optional[torch.Tensor],
         lm_head_indices: Optional[torch.Tensor] = None,
@@ -741,6 +749,7 @@ class Idefics2ForConditionalGeneration(nn.Module):
         pixel_attention_mask: Optional[torch.BoolTensor] = None,
         # Unused here
         image_sizes: Optional[torch.Tensor] = None,
+        adapter_data: Optional[torch.Tensor] = None,
     ):
         inputs_embeds = self.text_model.embed_tokens(input_ids)
         if pixel_values is not None:
@@ -818,10 +827,11 @@ class Idefics2ForConditionalGeneration(nn.Module):
             kv_cache=kv_cache,
             block_tables=block_tables,
             slots=slots,
-            input_lengths=input_lengths,
+            seqlen=seqlen,
             max_s=max_s,
             true_max_s=max_s,
             prefill_cache_indices=None,
+            adapter_data=adapter_data,
         )
         if lm_head_indices is not None:
             hidden_states = hidden_states[lm_head_indices]
diff --git a/server/text_generation_server/models/custom_modeling/idefics_image_processing.py b/server/text_generation_server/models/custom_modeling/idefics_image_processing.py
index e323d365..afb8e1f9 100644
--- a/server/text_generation_server/models/custom_modeling/idefics_image_processing.py
+++ b/server/text_generation_server/models/custom_modeling/idefics_image_processing.py
@@ -19,6 +19,7 @@ import numpy as np
 
 from PIL import Image
 
+import transformers
 from transformers.image_processing_utils import BaseImageProcessor, BatchFeature
 from transformers.image_transforms import (
     resize,
@@ -293,6 +294,4 @@ class IdeficsImageProcessor(BaseImageProcessor):
         return normalize(image, mean=mean, std=std, data_format=data_format, **kwargs)
 
 
-import transformers
-
 transformers.IdeficsImageProcessor = IdeficsImageProcessor
diff --git a/server/text_generation_server/models/custom_modeling/idefics_modeling.py b/server/text_generation_server/models/custom_modeling/idefics_modeling.py
index 786ef559..fc6becc4 100644
--- a/server/text_generation_server/models/custom_modeling/idefics_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/idefics_modeling.py
@@ -21,10 +21,8 @@
 from typing import List, Optional, Tuple, Union
 
 import torch
-import torch.nn.functional as F
 import torch.utils.checkpoint
 from torch import nn
-from torch.nn import CrossEntropyLoss
 
 from transformers import PreTrainedModel
 from transformers.activations import ACT2FN
@@ -33,13 +31,6 @@ from transformers.modeling_outputs import (
     CausalLMOutputWithPast,
     dataclass,
 )
-from transformers.modeling_utils import PretrainedConfig
-from transformers.utils import (
-    add_start_docstrings,
-    add_start_docstrings_to_model_forward,
-    logging,
-    replace_return_docstrings,
-)
 from text_generation_server.models.custom_modeling.idefics_config import IdeficsConfig
 from text_generation_server.models.custom_modeling.idefics_vision import (
     IdeficsVisionTransformer,
@@ -56,6 +47,7 @@ from text_generation_server.layers import (
 )
 from text_generation_server.layers.rotary import PositionRotaryEmbedding
 from text_generation_server.utils.import_utils import SYSTEM
+from loguru import logger
 
 if SYSTEM == "cuda":
     import dropout_layer_norm
@@ -237,7 +229,7 @@ class IdeficsDecoupledPartialTPEmbedding(nn.Module):
             prefix="model.embed_tokens", weights=weights
         )
         self.additional_weight = nn.Parameter(
-            weights.get_tensor(f"model.embed_tokens.additional_embedding.weight")
+            weights.get_tensor("model.embed_tokens.additional_embedding.weight")
         )
 
     def forward(self, input_ids):
@@ -359,7 +351,19 @@ class IdeficsRMSNorm(nn.Module):
         self.variance_epsilon = eps
 
     def forward(self, hidden_states, residual=None):
-        if hidden_states.shape[-1] > 8192:
+        if SYSTEM == "ipex":
+            import intel_extension_for_pytorch as ipex
+
+            out = ipex.llm.functional.add_rms_norm(
+                residual,
+                hidden_states,
+                self.weight,
+                None,
+                self.variance_epsilon,
+                residual is not None,
+            )
+            return out
+        elif hidden_states.shape[-1] > 8192:
             if residual is not None:
                 hidden_states += residual
             residual = hidden_states
@@ -499,7 +503,6 @@ class IdeficsAttention(nn.Module):
         # if not hasattr(nn.functional, "scaled_dot_product_attention"):
         #     raise ValueError("this model requires pytorch 2.0 or higher")
 
-        process_group = weights.process_group
         if self.num_heads % weights.process_group.size() != 0:
             raise ValueError(
                 f"`num_heads` must be divisible by `num_shards` (got `num_heads`: {self.num_heads} "
@@ -1024,7 +1027,7 @@ class IdeficsModel(IdeficsPreTrainedModel):
         if config.use_resampler:
             perceiver_config = config.perceiver_config
             self.perceiver_resampler = IdeficsPerceiverResampler(
-                prefix=f"model.perceiver_resampler",
+                prefix="model.perceiver_resampler",
                 config=config,
                 embed_dim=config.vision_config.embed_dim,
                 depth=perceiver_config.resampler_depth,
@@ -1052,7 +1055,7 @@ class IdeficsModel(IdeficsPreTrainedModel):
         # self.gradient_checkpointing = False
 
         self.norm = IdeficsRMSNorm(
-            prefix=f"model.norm", weights=weights, eps=config.rms_norm_eps
+            prefix="model.norm", weights=weights, eps=config.rms_norm_eps
         )
 
         # self.gradient_checkpointing = False
diff --git a/server/text_generation_server/models/custom_modeling/idefics_perceiver.py b/server/text_generation_server/models/custom_modeling/idefics_perceiver.py
index af44490b..6da8045b 100644
--- a/server/text_generation_server/models/custom_modeling/idefics_perceiver.py
+++ b/server/text_generation_server/models/custom_modeling/idefics_perceiver.py
@@ -169,7 +169,6 @@ class IdeficsPerceiverAttention(nn.Module):
 
         self.qk_scale = self.head_dim**-0.5
 
-        process_group = weights.process_group
         if n_heads % weights.process_group.size() != 0:
             raise ValueError(
                 f"`num_heads` must be divisible by `num_shards` (got `num_heads`: {n_heads} "
diff --git a/server/text_generation_server/models/custom_modeling/idefics_processing.py b/server/text_generation_server/models/custom_modeling/idefics_processing.py
index 7bba6977..ca61e27d 100644
--- a/server/text_generation_server/models/custom_modeling/idefics_processing.py
+++ b/server/text_generation_server/models/custom_modeling/idefics_processing.py
@@ -28,9 +28,6 @@ from transformers.tokenization_utils_base import (
     TruncationStrategy,
 )
 from transformers.utils import TensorType, is_torch_available
-from text_generation_server.models.custom_modeling.idefics_image_processing import (
-    IdeficsImageProcessor,
-)
 
 
 if is_torch_available():
diff --git a/server/text_generation_server/models/custom_modeling/idefics_vision.py b/server/text_generation_server/models/custom_modeling/idefics_vision.py
index 30c5997f..dd8f76bc 100644
--- a/server/text_generation_server/models/custom_modeling/idefics_vision.py
+++ b/server/text_generation_server/models/custom_modeling/idefics_vision.py
@@ -129,7 +129,6 @@ class IdeficsVisionAttention(nn.Module):
         self.scale = self.head_dim**-0.5
         self.dropout = config.attention_dropout
 
-        process_group = weights.process_group
         if self.num_heads % weights.process_group.size() != 0:
             raise ValueError(
                 f"`num_heads` must be divisible by `num_shards` (got `num_heads`: {self.num_heads} "
@@ -460,7 +459,6 @@ class IdeficsVisionTransformer(nn.Module):
     def __init__(self, prefix, config, weights):
         super().__init__()
         self.config = config
-        embed_dim = config.hidden_size
 
         self.embeddings = IdeficsVisionEmbeddings(
             prefix=f"{prefix}.embeddings", config=config, weights=weights
diff --git a/server/text_generation_server/models/custom_modeling/mllama.py b/server/text_generation_server/models/custom_modeling/mllama.py
new file mode 100644
index 00000000..73536bd6
--- /dev/null
+++ b/server/text_generation_server/models/custom_modeling/mllama.py
@@ -0,0 +1,995 @@
+# coding=utf-8
+# Copyright 2024 the HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""PyTorch Mllama model."""
+
+from typing import Optional, Tuple, List
+
+import torch
+import torch.utils.checkpoint
+from torch import nn
+import flash_attn_2_cuda
+
+from transformers.activations import ACT2FN
+import torch.nn.functional as F
+
+from text_generation_server.layers import (
+    TensorParallelColumnLinear,
+    TensorParallelEmbedding,
+    TensorParallelRowLinear,
+    FastLinear,
+)
+from text_generation_server.layers.attention import (
+    Seqlen,
+)
+from text_generation_server.models.custom_modeling.flash_llama_modeling import (
+    FlashLlamaForCausalLM,
+)
+
+
+def _prepare_aspect_ratio_attention_mask(
+    aspect_ratio_mask: torch.Tensor,
+    num_patches: int,
+    target_length: int,
+    dtype: torch.dtype,
+) -> torch.Tensor:
+    # Expand aspect ratio mask to target_length
+    batch_size, max_num_tiles = aspect_ratio_mask.shape
+    attention_mask = aspect_ratio_mask.view(batch_size, max_num_tiles, 1, 1).to(dtype)
+    attention_mask = attention_mask.repeat(1, 1, target_length, 1)
+
+    # Mask padding patches
+    pad_patches = target_length - num_patches
+    attention_mask[:, :, -pad_patches:] = 0
+
+    # Invert the mask (0 -> 1, 1 -> 0)
+    attention_mask = 1 - attention_mask
+
+    # Reshape to 2D and create 4D attention mask
+    # (batch_size, 1, max_num_tiles * target_length, max_num_tiles * target_length)
+    attention_mask = attention_mask.reshape(
+        batch_size, max_num_tiles * target_length, 1
+    )
+    attention_mask = (
+        attention_mask @ attention_mask.transpose(-1, -2) * torch.finfo(dtype).min
+    )
+    attention_mask = attention_mask.unsqueeze(1)
+
+    return attention_mask
+
+
+# Copied from transformers.models.llama.modeling_llama._prepare_4d_causal_attention_mask_with_cache_position
+def _prepare_4d_causal_attention_mask_with_cache_position(
+    attention_mask: torch.Tensor,
+    sequence_length: int,
+    target_length: int,
+    dtype: torch.dtype,
+    device: torch.device,
+    min_dtype: float,
+    cache_position: torch.Tensor,
+    batch_size: int,
+):
+    """
+    Creates a causal 4D mask of shape `(batch_size, 1, query_length, key_value_length)` from a 2D mask of shape
+    `(batch_size, key_value_length)`, or if the input `attention_mask` is already 4D, do nothing.
+
+    Args:
+        attention_mask (`torch.Tensor`):
+            A 2D attention mask of shape `(batch_size, key_value_length)` or a 4D attention mask of shape `(batch_size, 1, query_length, key_value_length)`.
+        sequence_length (`int`):
+            The sequence length being processed.
+        target_length (`int`):
+            The target length: when generating with static cache, the mask should be as long as the static cache, to account for the 0 padding, the part of the cache that is not filled yet.
+        dtype (`torch.dtype`):
+            The dtype to use for the 4D attention mask.
+        device (`torch.device`):
+            The device to plcae the 4D attention mask on.
+        min_dtype (`float`):
+            The minimum value representable with the dtype `dtype`.
+        cache_position (`torch.Tensor`):
+            Indices depicting the position of the input sequence tokens in the sequence.
+        batch_size (`torch.Tensor`):
+            Batch size.
+    """
+    if attention_mask is not None and attention_mask.dim() == 4:
+        # In this case we assume that the mask comes already in inverted form and requires no inversion or slicing.
+        causal_mask = attention_mask
+    else:
+        causal_mask = torch.full(
+            (sequence_length, target_length),
+            fill_value=min_dtype,
+            dtype=dtype,
+            device=device,
+        )
+        if sequence_length != 1:
+            causal_mask = torch.triu(causal_mask, diagonal=1)
+        causal_mask *= torch.arange(
+            target_length, device=device
+        ) > cache_position.reshape(-1, 1)
+        causal_mask = causal_mask[None, None, :, :].expand(batch_size, 1, -1, -1)
+        if attention_mask is not None:
+            causal_mask = (
+                causal_mask.clone()
+            )  # copy to contiguous memory for in-place edit
+            mask_length = attention_mask.shape[-1]
+            padding_mask = (
+                causal_mask[:, :, :, :mask_length] + attention_mask[:, None, None, :]
+            )
+            padding_mask = padding_mask == 0
+            causal_mask[:, :, :, :mask_length] = causal_mask[
+                :, :, :, :mask_length
+            ].masked_fill(padding_mask, min_dtype)
+
+    return causal_mask
+
+
+def _prepare_cross_attention_mask(
+    cross_attention_mask: torch.Tensor,
+    num_vision_tokens: int,
+    dtype: str,
+) -> Tuple[torch.Tensor, torch.Tensor]:
+    # reshape so it can be used by attn module
+    batch_size, text_total_length, *_ = cross_attention_mask.shape
+    cross_attention_mask = cross_attention_mask.repeat_interleave(
+        num_vision_tokens, dim=3
+    )
+    cross_attention_mask = cross_attention_mask.view(batch_size, text_total_length, -1)
+    cross_attention_mask = cross_attention_mask.unsqueeze(1)
+
+    # invert the mask
+    inverted_cross_attn_mask = (1.0 - cross_attention_mask).to(dtype)
+    cross_attention_mask = inverted_cross_attn_mask.masked_fill(
+        inverted_cross_attn_mask.to(torch.bool), torch.finfo(dtype).min
+    )
+
+    # apply full-row bias, which return 4D tensor of shape [B, H, S1, 1] where value is 0 if the a full row in cross attn mask's
+    # last dimension contains negative infinity values, otherwise it's 1
+    negative_inf_value = torch.finfo(dtype).min
+    full_text_row_masked_out_mask = (
+        (cross_attention_mask != negative_inf_value)
+        .any(dim=-1)
+        .type_as(cross_attention_mask)[..., None]
+    )
+    cross_attention_mask *= full_text_row_masked_out_mask
+
+    return cross_attention_mask, full_text_row_masked_out_mask
+
+
+# Copied from transformers.models.clip.modeling_clip.CLIPMLP with CLIP->MllamaVision
+class MllamaVisionMLP(nn.Module):
+    def __init__(self, *, prefix, config, weights):
+        super().__init__()
+        self.config = config
+        self.activation_fn = ACT2FN[config.hidden_act]
+        self.fc1 = TensorParallelColumnLinear.load(
+            prefix=f"{prefix}.fc1", weights=weights, config=config, bias=True
+        )
+        self.fc2 = TensorParallelRowLinear.load(
+            prefix=f"{prefix}.fc2", weights=weights, config=config, bias=True
+        )
+
+    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+        hidden_states = self.fc1(hidden_states)
+        hidden_states = self.activation_fn(hidden_states)
+        hidden_states = self.fc2(hidden_states)
+        return hidden_states
+
+
+class MllamaVisionSdpaAttention(nn.Module):
+    def __init__(self, *, prefix, config, weights):
+        super().__init__()
+
+        self.embed_dim = config.hidden_size
+        self.head_dim = config.hidden_size // config.attention_heads
+        self.num_heads = config.attention_heads // weights.process_group.size()
+
+        self.qkv_proj = TensorParallelColumnLinear.load_multi(
+            config,
+            prefixes=[f"{prefix}.q_proj", f"{prefix}.k_proj", f"{prefix}.v_proj"],
+            dim=0,
+            weights=weights,
+            bias=False,
+        )
+        self.o_proj = TensorParallelRowLinear.load(
+            config,
+            prefix=f"{prefix}.o_proj",
+            weights=weights,
+            bias=False,
+        )
+
+    def forward(
+        self,
+        hidden_state: torch.Tensor,
+        attention_mask: Optional[torch.Tensor] = None,
+    ) -> torch.Tensor:
+        qkv = self.qkv_proj(hidden_state)
+        query, key, value = qkv.split(
+            [
+                self.head_dim * self.num_heads,
+                self.head_dim * self.num_heads,
+                self.head_dim * self.num_heads,
+            ],
+            dim=2,
+        )
+
+        batch_size, q_seq_len, _ = query.shape
+        _, kv_seq_len, _ = key.shape
+
+        query = query.view(batch_size, q_seq_len, self.num_heads, self.head_dim)
+        key = key.view(batch_size, kv_seq_len, self.num_heads, self.head_dim)
+        value = value.view(batch_size, kv_seq_len, self.num_heads, self.head_dim)
+
+        query = query.transpose(1, 2)
+        key = key.transpose(1, 2)
+        value = value.transpose(1, 2)
+
+        attn_output = F.scaled_dot_product_attention(
+            query, key, value, attn_mask=attention_mask
+        )
+
+        attn_output = attn_output.transpose(1, 2).contiguous()
+        attn_output = attn_output.reshape(batch_size, q_seq_len, -1)
+
+        output = self.o_proj(attn_output)
+        return output
+
+
+class MllamaVisionEncoderLayer(nn.Module):
+    def __init__(self, *, prefix, config, weights, is_gated: bool):
+        super().__init__()
+
+        self.hidden_size = config.hidden_size
+        self.num_attention_heads = config.attention_heads
+        self.is_gated = is_gated
+        self.intermediate_size = config.intermediate_size
+
+        self.self_attn = MllamaVisionSdpaAttention(
+            prefix=f"{prefix}.self_attn", config=config, weights=weights
+        )
+        self.mlp = MllamaVisionMLP(
+            prefix=f"{prefix}.mlp", config=config, weights=weights
+        )
+
+        self.input_layernorm = nn.LayerNorm.load(
+            prefix=f"{prefix}.input_layernorm", weights=weights, eps=1e-05
+        )
+        self.post_attention_layernorm = nn.LayerNorm.load(
+            prefix=f"{prefix}.post_attention_layernorm", weights=weights, eps=1e-05
+        )
+
+        # there used to be an if else here, no code path
+        if is_gated:
+            self.gate_attn = nn.Parameter(
+                weights.get_tensor(f"{prefix}.gate_attn"), requires_grad=False
+            )
+            self.gate_ffn = nn.Parameter(
+                weights.get_tensor(f"{prefix}.gate_ffn"), requires_grad=False
+            )
+
+    def forward(
+        self,
+        hidden_state: torch.Tensor,
+        attention_mask: Optional[torch.Tensor] = None,
+    ):
+        # Self Attention
+        residual = hidden_state
+        hidden_state = self.input_layernorm(hidden_state)
+        hidden_state = self.self_attn(hidden_state, attention_mask=attention_mask)
+        gate_attn = 1 if not self.is_gated else self.gate_attn.tanh()
+        hidden_state = residual + gate_attn * hidden_state
+
+        # Feed forward
+        residual = hidden_state
+        hidden_state = self.post_attention_layernorm(hidden_state)
+        hidden_state = self.mlp(hidden_state)
+        gate_ffn = 1 if not self.is_gated else self.gate_ffn.tanh()
+        hidden_state = residual + gate_ffn * hidden_state
+        return hidden_state
+
+
+class MllamaVisionEncoder(nn.Module):
+    def __init__(self, *, prefix, config, weights, is_gated: bool, num_layers: int):
+        super().__init__()
+        self.config = config
+        self.layers = [
+            MllamaVisionEncoderLayer(
+                prefix=f"{prefix}.layers.{i}",
+                config=config,
+                weights=weights,
+                is_gated=is_gated,
+            )
+            for i in range(num_layers)
+        ]
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: Optional[torch.Tensor] = None,
+    ):
+        encoder_states = [hidden_states]
+        for encoder_layer in self.layers:
+            layer_outputs = encoder_layer(
+                hidden_states,
+                attention_mask,
+            )
+
+            hidden_states = layer_outputs
+            encoder_states.append(hidden_states)
+
+        return hidden_states, encoder_states
+
+
+class MllamaPrecomputedAspectRatioEmbedding(nn.Module):
+    def __init__(self, *, prefix, config, weights):
+        super().__init__()
+        self.max_num_tiles = config.max_num_tiles
+        self.hidden_size = config.hidden_size
+        self.max_aspect_ratio_id = config.max_aspect_ratio_id
+
+        self.embedding = TensorParallelEmbedding(
+            prefix=f"{prefix}.embedding", weights=weights
+        )
+        self.gate = nn.Parameter(
+            weights.get_tensor(f"{prefix}.gate"), requires_grad=False
+        )
+
+    def forward(
+        self, hidden_state: torch.Tensor, aspect_ratio_ids: torch.Tensor
+    ) -> torch.Tensor:
+        embeddings = self.embedding(aspect_ratio_ids)
+        embeddings = embeddings.reshape(-1, self.max_num_tiles, 1, self.hidden_size)
+
+        # Always gated.
+        embeddings = embeddings * self.gate.tanh()
+
+        hidden_state = hidden_state + embeddings
+        return hidden_state
+
+
+class MllamaPrecomputedPositionEmbedding(nn.Module):
+    def __init__(self, *, prefix, config, weights):
+        super().__init__()
+        self.max_num_tiles = config.max_num_tiles
+        self.max_aspect_ratio_id = config.max_aspect_ratio_id
+        self.num_patches = (config.image_size // config.patch_size) ** 2 + 1
+        self.hidden_size = config.hidden_size
+        self.scale = config.hidden_size**-0.5
+
+        self.gate = nn.Parameter(
+            weights.get_tensor(f"{prefix}.gate"), requires_grad=False
+        )
+
+        # position embedding
+        embedding = nn.Parameter(
+            weights.get_tensor(f"{prefix}.embedding"), requires_grad=False
+        )
+        self.gated_position_embedding = (1 - self.gate.tanh()) * embedding
+        self.tile_embedding = TensorParallelEmbedding(
+            prefix=f"{prefix}.tile_embedding", weights=weights
+        )
+
+    def forward(
+        self, hidden_state: torch.Tensor, aspect_ratio_ids: torch.Tensor
+    ) -> torch.Tensor:
+        # position embeddings
+        hidden_state = hidden_state + self.gated_position_embedding.view(
+            1, 1, self.num_patches, self.hidden_size
+        )
+
+        # precomputed tile position embeddings
+        tile_position_embedding = self.tile_embedding(aspect_ratio_ids)
+        batch_size = hidden_state.shape[0]
+        tile_position_embedding = tile_position_embedding.reshape(
+            batch_size, self.max_num_tiles, self.num_patches, self.hidden_size
+        )
+        gated_tile_position_embedding = self.gate.tanh() * tile_position_embedding
+        hidden_state = hidden_state + gated_tile_position_embedding
+
+        return hidden_state
+
+
+class MllamaVisionModel(nn.Module):
+    def __init__(self, *, prefix, config, weights):
+        super().__init__()
+        self.image_size = config.image_size
+        self.patch_size = config.patch_size
+        self.max_num_tiles = config.max_num_tiles
+        self.hidden_size = config.hidden_size
+        self.num_channels = config.num_channels
+        self.intermediate_layers_indices = config.intermediate_layers_indices
+
+        self.num_patches = (self.image_size // self.patch_size) ** 2 + 1
+        self.scale = config.hidden_size**-0.5
+        self.dtype = weights.dtype
+
+        self.patch_embedding = nn.Conv2d(
+            in_channels=config.num_channels,
+            out_channels=self.hidden_size,
+            kernel_size=self.patch_size,
+            stride=self.patch_size,
+            padding="valid",
+            bias=False,
+        )
+        self.patch_embedding.weight = nn.Parameter(
+            weights.get_tensor(f"{prefix}.patch_embedding.weight"), requires_grad=False
+        )
+
+        self.class_embedding = nn.Parameter(
+            weights.get_tensor(f"{prefix}.class_embedding"), requires_grad=False
+        )
+
+        self.gated_positional_embedding = MllamaPrecomputedPositionEmbedding(
+            prefix=f"{prefix}.gated_positional_embedding",
+            config=config,
+            weights=weights,
+        )
+
+        self.pre_tile_positional_embedding = MllamaPrecomputedAspectRatioEmbedding(
+            prefix=f"{prefix}.pre_tile_positional_embedding",
+            config=config,
+            weights=weights,
+        )
+        self.post_tile_positional_embedding = MllamaPrecomputedAspectRatioEmbedding(
+            prefix=f"{prefix}.post_tile_positional_embedding",
+            config=config,
+            weights=weights,
+        )
+
+        ## layer norms
+        self.layernorm_pre = nn.LayerNorm.load(
+            prefix=f"{prefix}.layernorm_pre",
+            weights=weights,
+            # torch default
+            eps=1e-05,
+        )
+        self.layernorm_post = nn.LayerNorm.load(
+            prefix=f"{prefix}.layernorm_post",
+            weights=weights,
+            # torch default
+            eps=1e-05,
+        )
+
+        ## encoders
+        self.transformer = MllamaVisionEncoder(
+            prefix=f"{prefix}.transformer",
+            config=config,
+            weights=weights,
+            is_gated=False,
+            num_layers=config.num_hidden_layers,
+        )
+        self.global_transformer = MllamaVisionEncoder(
+            prefix=f"{prefix}.global_transformer",
+            config=config,
+            weights=weights,
+            is_gated=True,
+            num_layers=config.num_global_layers,
+        )
+
+    def apply_class_embedding(self, hidden_state: torch.Tensor) -> torch.Tensor:
+        batch_size, _, hidden_size = hidden_state.shape
+        class_embedding = self.class_embedding.expand(batch_size, 1, hidden_size)
+        hidden_state = torch.cat([class_embedding, hidden_state], dim=1)
+        return hidden_state
+
+    def forward(
+        self,
+        pixel_values: torch.Tensor,
+        aspect_ratio_ids: torch.Tensor,
+        attention_mask: torch.Tensor,
+    ) -> torch.Tensor:
+        batch_size, num_concurrent_media, num_tiles, num_channels, height, width = (
+            pixel_values.shape
+        )
+
+        pixel_values = pixel_values.reshape(
+            batch_size * num_concurrent_media * num_tiles, num_channels, height, width
+        )
+        aspect_ratio_ids = aspect_ratio_ids.reshape(
+            batch_size * num_concurrent_media, -1
+        )
+
+        # patch embedding
+        patch_embeds = self.patch_embedding(pixel_values)
+        hidden_state = patch_embeds.flatten(2).transpose(1, 2)
+
+        # tile embeddings
+        _, num_patches, dim = hidden_state.shape
+        hidden_state = hidden_state.reshape(
+            batch_size * num_concurrent_media, num_tiles, -1, dim
+        )
+        hidden_state = self.pre_tile_positional_embedding(
+            hidden_state, aspect_ratio_ids
+        )
+
+        # apply cls token
+        hidden_state = hidden_state.reshape(
+            batch_size * num_concurrent_media * num_tiles, num_patches, dim
+        )
+        hidden_state = self.apply_class_embedding(hidden_state)
+        num_patches += 1
+
+        # apply position embeddings
+        hidden_state = hidden_state.reshape(
+            batch_size * num_concurrent_media, num_tiles, num_patches, dim
+        )
+        hidden_state = self.gated_positional_embedding(hidden_state, aspect_ratio_ids)
+
+        # apply encoder
+        hidden_state = self.layernorm_pre(hidden_state)
+
+        # Compute the number of tokens to pad
+        num_padding_patches = (8 - (hidden_state.shape[-2] % 8)) % 8
+        # Compute padding tuple for pad function
+        padding = (
+            0,
+            0,
+            0,
+            num_padding_patches,
+        )  # (pad_left, pad_right, pad_left for dim -2, pad_right for dim -2)
+        # Pad the tensor
+        hidden_state = F.pad(hidden_state, padding, mode="constant", value=0)
+        slice_index = -num_padding_patches if num_padding_patches > 0 else None
+
+        if attention_mask is not None:
+            attention_mask = attention_mask.reshape(
+                batch_size * num_concurrent_media, -1
+            )
+            attention_mask = _prepare_aspect_ratio_attention_mask(
+                aspect_ratio_mask=attention_mask,
+                num_patches=self.num_patches,
+                target_length=hidden_state.shape[2],
+                dtype=self.dtype,
+            )
+
+        hidden_state = hidden_state.view(batch_size * num_concurrent_media, -1, dim)
+        hidden_state, all_intermediate_hidden_states = self.transformer(
+            hidden_state,
+            attention_mask=attention_mask,
+        )
+        intermediate_hidden_states = [
+            hidden_state
+            for idx, hidden_state in enumerate(all_intermediate_hidden_states)
+            if idx in self.intermediate_layers_indices
+        ]
+        intermediate_hidden_states = torch.stack(intermediate_hidden_states, dim=-1)
+
+        # apply global encoder
+        hidden_state = self.layernorm_post(hidden_state)
+        hidden_state = hidden_state.reshape(
+            batch_size * num_concurrent_media,
+            num_tiles,
+            num_patches + num_padding_patches,
+            dim,
+        )
+        hidden_state = self.post_tile_positional_embedding(
+            hidden_state, aspect_ratio_ids
+        )
+        hidden_state = hidden_state.reshape(
+            batch_size * num_concurrent_media,
+            num_tiles * (num_patches + num_padding_patches),
+            dim,
+        )
+        hidden_state, _ = self.global_transformer(
+            hidden_state, attention_mask=attention_mask
+        )
+        hidden_state = hidden_state.reshape(
+            batch_size * num_concurrent_media,
+            num_tiles,
+            num_patches + num_padding_patches,
+            dim,
+        )
+        hidden_state = hidden_state[:, :, :slice_index]
+
+        # adding intermediate layer outputs
+        hidden_state = hidden_state.reshape(
+            batch_size, num_concurrent_media, num_tiles, num_patches, dim
+        )
+        intermediate_hidden_states = intermediate_hidden_states.reshape(
+            batch_size * num_concurrent_media,
+            num_tiles,
+            num_patches + num_padding_patches,
+            -1,
+        )
+        intermediate_hidden_states = intermediate_hidden_states[:, :, :slice_index]
+        intermediate_hidden_states = intermediate_hidden_states.reshape(
+            batch_size, num_concurrent_media, num_tiles, num_patches, -1
+        )
+        hidden_state = torch.cat([hidden_state, intermediate_hidden_states], dim=-1)
+        return hidden_state
+
+
+class MllamaTextCrossAttention(nn.Module):
+    """Multi-headed attention from 'Attention Is All You Need' paper"""
+
+    def __init__(self, *, prefix, config, weights, layer_idx):
+        super().__init__()
+        self.config = config
+        self.num_heads = self.config.num_attention_heads
+        self.num_key_value_heads = self.config.num_key_value_heads
+        self.dropout = config.dropout
+        self.hidden_size = config.hidden_size
+        self.head_size = config.hidden_size // self.num_heads
+        self.num_key_value_groups = self.num_heads // self.num_key_value_heads
+        self.layer_idx = layer_idx
+
+        self.num_heads = self.num_heads // weights.process_group.size()
+        self.num_key_value_heads = (
+            self.num_key_value_heads // weights.process_group.size()
+        )
+
+        self.q_proj = TensorParallelColumnLinear.load(
+            config,
+            prefix=f"{prefix}.q_proj",
+            weights=weights,
+            bias=False,
+        )
+        self.k_proj = TensorParallelColumnLinear.load(
+            config,
+            prefix=f"{prefix}.k_proj",
+            weights=weights,
+            bias=False,
+        )
+        self.v_proj = TensorParallelColumnLinear.load(
+            config,
+            prefix=f"{prefix}.v_proj",
+            weights=weights,
+            bias=False,
+        )
+        self.o_proj = TensorParallelRowLinear.load(
+            config,
+            prefix=f"{prefix}.o_proj",
+            weights=weights,
+            bias=False,
+        )
+
+        self.q_norm = MllamaTextRMSNorm.load(
+            prefix=f"{prefix}.q_norm", weights=weights, eps=config.rms_norm_eps
+        )
+        self.k_norm = MllamaTextRMSNorm.load(
+            prefix=f"{prefix}.k_norm", weights=weights, eps=config.rms_norm_eps
+        )
+        self.softmax_scale = self.head_size**-0.5
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        cross_attention_states: Optional[torch.Tensor] = None,
+        # past_key_value=None,
+        # attention_mask: Optional[torch.Tensor] = None,
+        # cache_position: Optional[torch.LongTensor] = None,
+    ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
+        """Input shape: Batch x Time x Channel"""
+        # hidden_states = hidden_states.unsqueeze(0)
+        # bsz, q_len, _ = hidden_states.size()
+        query_states = self.q_proj(hidden_states)
+        query_states = query_states.view(-1, self.num_heads, self.head_size)
+        query_states = self.q_norm(query_states)
+
+        (
+            cross_attention_states,
+            cu_seqlen_q,
+            cu_seqlen_k,
+            max_q,
+            max_k,
+            indices,
+        ) = cross_attention_states
+
+        key_states = self.k_proj(cross_attention_states)
+        value_states = self.v_proj(cross_attention_states)
+        key_states = key_states.view(-1, self.num_key_value_heads, self.head_size)
+        value_states = value_states.view(-1, self.num_key_value_heads, self.head_size)
+        key_states = self.k_norm(key_states)
+
+        # key_states = key_states.repeat(1, self.num_key_value_groups, 1)
+        # value_states = value_states.repeat(1, self.num_key_value_groups, 1)
+
+        causal = False
+        # logger.info(
+        #     f"Q: {query_states.shape} -K {key_states.shape} - V{value_states.shape}"
+        # )
+        attn_output = flash_attn_2_cuda.varlen_fwd(
+            query_states,
+            key_states,
+            value_states,
+            None,
+            cu_seqlen_q,
+            cu_seqlen_k,
+            None,
+            None,
+            None,  # block_tables
+            None,
+            max_q,
+            max_k,
+            0.0,
+            self.softmax_scale,
+            False,
+            causal,  # Causal
+            -1,  # window_size_left,
+            -1,
+            0.0,  # softcap
+            False,
+            None,
+        )[0]
+        attn_output = self.o_proj(attn_output.view(-1, self.num_heads * self.head_size))
+
+        return attn_output
+
+
+# Copied from transformers.models.gemma2.modeling_gemma2.Gemma2MLP with Gemma2->MllamaText
+class MllamaTextMLP(nn.Module):
+    def __init__(self, *, prefix, config, weights):
+        super().__init__()
+        self.config = config
+        self.hidden_size = config.hidden_size
+        self.intermediate_size = (
+            config.intermediate_size // weights.process_group.size()
+        )
+        self.gate_up_proj = TensorParallelColumnLinear.load_multi(
+            config,
+            prefixes=[f"{prefix}.gate_proj", f"{prefix}.up_proj"],
+            weights=weights,
+            dim=0,
+            bias=False,
+        )
+        self.down_proj = TensorParallelRowLinear.load(
+            config,
+            prefix=f"{prefix}.down_proj",
+            weights=weights,
+            bias=False,
+        )
+        self.act_fn = ACT2FN[config.hidden_act]
+
+    def forward(self, x):
+        shape = x.shape
+        gate_up_states = self.gate_up_proj(x)
+        gate_up_states = gate_up_states.view(*shape[:-1], 2, self.intermediate_size)
+        result = self.down_proj(
+            self.act_fn(gate_up_states[:, 0]) * gate_up_states[:, 1]
+        )
+        return result
+
+
+class FlashLlamaCrossLayer(torch.nn.Module):
+    """Cross-attention transformer block with tanh-gated attention and feedforward."""
+
+    def __init__(self, *, prefix, config, weights, index) -> None:
+        layer_idx = index
+        super().__init__()
+        self.cross_attn = MllamaTextCrossAttention(
+            prefix=f"{prefix}.cross_attn",
+            config=config,
+            weights=weights,
+            layer_idx=layer_idx,
+        )
+
+        self.input_layernorm = MllamaTextRMSNorm.load(
+            prefix=f"{prefix}.input_layernorm", weights=weights, eps=config.rms_norm_eps
+        )
+        self.cross_attn_attn_gate = torch.nn.Parameter(
+            weights.get_tensor(f"{prefix}.cross_attn_attn_gate"), requires_grad=False
+        )
+
+        self.mlp = MllamaTextMLP(prefix=f"{prefix}.mlp", config=config, weights=weights)
+        self.post_attention_layernorm = MllamaTextRMSNorm.load(
+            prefix=f"{prefix}.post_attention_layernorm",
+            weights=weights,
+            eps=config.rms_norm_eps,
+        )
+        self.cross_attn_mlp_gate = torch.nn.Parameter(
+            weights.get_tensor(f"{prefix}.cross_attn_mlp_gate"), requires_grad=False
+        )
+        self.layer_idx = layer_idx
+
+    def forward(
+        self,
+        hidden_states,
+        residual,
+        cos,
+        sin,
+        cu_seqlen_prefill,
+        kv_cache,
+        block_tables,
+        slots,
+        seqlen,
+        max_s,
+        adapter_data,
+        cross_attention_states,  # [ IB, ...]
+    ) -> Tuple[torch.Tensor, torch.Tensor]:
+        if cross_attention_states is None:
+            return hidden_states, residual
+        if residual is not None:
+            hidden_states += residual
+
+        indices = cross_attention_states[-1]
+        out_hidden_states = hidden_states[:]
+        if len(indices) > 0:
+            assert max(indices) < hidden_states.shape[0]
+        hidden_states = hidden_states[indices]
+        residual = hidden_states
+        hidden_states = self.input_layernorm(hidden_states)
+
+        hidden_states = self.cross_attn(
+            hidden_states=hidden_states,
+            # attention_mask=cross_attention_mask,
+            cross_attention_states=cross_attention_states,
+        )
+        hidden_states = residual + self.cross_attn_attn_gate.tanh() * hidden_states
+
+        residual = hidden_states
+        hidden_states = self.post_attention_layernorm(hidden_states)
+        hidden_states = self.mlp(hidden_states)
+        hidden_states = residual + self.cross_attn_mlp_gate.tanh() * hidden_states
+
+        out_hidden_states[indices] = hidden_states
+        hidden_states = out_hidden_states
+
+        return hidden_states, None
+
+
+# Copied from transformers.models.llama.modeling_llama.LlamaRMSNorm with Llama->MllamaText
+class MllamaTextRMSNorm(nn.Module):
+    def __init__(self, weight, eps):
+        super().__init__()
+        self.weight = weight
+        self.variance_epsilon = eps
+
+    @classmethod
+    def load(cls, *, prefix, weights, eps):
+        weight = nn.Parameter(
+            weights.get_tensor(f"{prefix}.weight"), requires_grad=False
+        )
+        return cls(weight=weight, eps=eps)
+
+    def forward(self, hidden_states):
+        input_dtype = hidden_states.dtype
+        hidden_states = hidden_states.to(torch.float32)
+        variance = hidden_states.pow(2).mean(-1, keepdim=True)
+        hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
+        return self.weight * hidden_states.to(input_dtype)
+
+    def extra_repr(self):
+        return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}"
+
+
+class MllamaForConditionalGeneration(nn.Module):
+    def __init__(self, prefix, config, weights):
+        super().__init__()
+        config.vision_config.quantize = None
+        config.vision_config.speculator = config.speculator
+        config.text_config.quantize = config.quantize
+        config.text_config.speculator = config.speculator
+        config.text_config._attn_implementation = "sdpa"
+        self.hidden_size = config.text_config.hidden_size
+        self.vision_model = MllamaVisionModel(
+            prefix="vision_model", config=config.vision_config, weights=weights
+        )
+        self.multi_modal_projector = FastLinear.load(
+            prefix="multi_modal_projector", config=config, weights=weights, bias=True
+        )
+        self.text_model = FlashLlamaForCausalLM(
+            prefix="language_model", config=config.text_config, weights=weights
+        )
+        self.config = config
+        self.dtype = weights.dtype
+        self.device = weights.device
+
+    def vision_forward(self, pixel_values, aspect_ratio_ids, aspect_ratio_mask):
+        if aspect_ratio_ids is None:
+            raise ValueError(
+                "`aspect_ratio_ids` must be provided if `pixel_values` is provided"
+            )
+        # logger.info(f"PIxel values {pixel_values.shape}")
+        batch_size = pixel_values.shape[0]
+        vision_states = self.vision_model(
+            pixel_values, aspect_ratio_ids, aspect_ratio_mask
+        )
+        cross_attention_states = self.multi_modal_projector(vision_states).reshape(
+            -1, vision_states.shape[-2], self.hidden_size
+        )
+        _, _, h = cross_attention_states.shape
+        cross_attention_states = cross_attention_states.view(batch_size, -1, h)
+        # logger.info(f"cross {cross_attention_states.shape}")
+        return cross_attention_states
+
+    def forward(
+        self,
+        input_ids: torch.Tensor,
+        position_ids: torch.Tensor,
+        cu_seqlen_prefill: Optional[torch.Tensor],
+        kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
+        block_tables: torch.Tensor,
+        slots: torch.Tensor,
+        seqlen: Seqlen,
+        max_s: int,
+        prefill_cache_indices: Optional[torch.Tensor],
+        lm_head_indices: Optional[torch.Tensor],
+        adapter_data: Optional[torch.Tensor] = None,
+        # XXX: Putting these as optional so that the cuda warmup calls can go through.
+        cross_attention_states: Optional[torch.Tensor] = None,
+        image_indices=None,
+    ):
+        if cross_attention_states is not None:
+            seqlen_q = len(image_indices)
+            n_images = cross_attention_states.shape[0]
+            seqlen_k = cross_attention_states.shape[1]
+            device = cross_attention_states.device
+            if cu_seqlen_prefill is not None:
+                offset = 0
+                cu_q = []
+                indices = []
+                for index in image_indices:
+                    cu_q.append(offset)
+                    length = seqlen.input_lengths[index].item()
+                    assert index < seqlen.cu_seqlen_q.shape[0]
+                    input_ids_offset = seqlen.cu_seqlen_q[index]
+                    indices.extend(range(input_ids_offset, input_ids_offset + length))
+                    offset += length
+                cu_q.append(offset)
+                cu_seqlen_q = torch.Tensor(cu_q).to(device=device, dtype=torch.int32)
+
+                assert max(indices) < input_ids.shape[0]
+
+                cu_seqlen_k = (
+                    torch.arange(
+                        n_images + 1,
+                        device=device,
+                        dtype=torch.int32,
+                    )
+                    * seqlen_k
+                )
+                max_q = cu_seqlen_q[-1].item()
+                max_k = seqlen_k
+            else:
+                cu_seqlen_q = torch.arange(
+                    seqlen_q + 1, device=device, dtype=torch.int32
+                )
+                seqlen_k = cross_attention_states.shape[1]
+                n_images = cross_attention_states.shape[0]
+                cu_seqlen_k = (
+                    torch.arange(
+                        n_images + 1,
+                        device=device,
+                        dtype=torch.int32,
+                    )
+                    * seqlen_k
+                )
+                max_q = seqlen_q
+                max_k = seqlen_k
+                indices = image_indices[:]
+
+            cross_attention_states = (
+                cross_attention_states,
+                cu_seqlen_q,
+                cu_seqlen_k,
+                max_q,
+                max_k,
+                indices,
+            )
+
+        outputs = self.text_model(
+            input_ids=input_ids,
+            position_ids=position_ids,
+            cu_seqlen_prefill=cu_seqlen_prefill,
+            kv_cache=kv_cache,
+            block_tables=block_tables,
+            slots=slots,
+            seqlen=seqlen,
+            max_s=max_s,
+            prefill_cache_indices=prefill_cache_indices,
+            lm_head_indices=lm_head_indices,
+            adapter_data=adapter_data,
+            cross_attention_states=cross_attention_states,
+        )
+
+        return outputs
diff --git a/server/text_generation_server/models/custom_modeling/mpt_modeling.py b/server/text_generation_server/models/custom_modeling/mpt_modeling.py
index f7981bf5..988a74a3 100644
--- a/server/text_generation_server/models/custom_modeling/mpt_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/mpt_modeling.py
@@ -4,7 +4,6 @@ Inspired by https://github.com/karpathy/minGPT/blob/master/mingpt/model.py
 """
 
 import math
-import os
 import warnings
 from typing import List, Optional, Tuple, Union
 import torch
@@ -75,7 +74,7 @@ def load_col(config, prefix, weights, bias):
         bias = bias.to(device=weights.device)
     else:
         bias = None
-    linear = get_linear(weight, bias, config.quantize)
+    linear = get_linear(weight, bias)
     return TensorParallelColumnLinear(linear)
 
 
@@ -194,7 +193,7 @@ def flash_attn_fn(
 ):
     try:
         from flash_attn import bert_padding, flash_attn_interface
-    except:
+    except Exception:
         raise RuntimeError("Please install flash-attn==1.0.3.post0")
     check_valid_inputs(query, key, value)
     if past_key_value is not None:
@@ -207,7 +206,7 @@ def flash_attn_fn(
         _s_k = max(0, attn_bias.size(3) - key.size(1))
         attn_bias = attn_bias[:, :, _s_q:, _s_k:]
     if attn_bias is not None:
-        raise NotImplementedError(f"attn_bias not implemented for flash attn.")
+        raise NotImplementedError("attn_bias not implemented for flash attn.")
     (batch_size, seqlen) = query.shape[:2]
     if key_padding_mask is None:
         key_padding_mask = torch.ones_like(key[:, :, 0], dtype=torch.bool)
@@ -269,13 +268,13 @@ def triton_flash_attn_fn(
 ):
     try:
         from .flash_attn_triton import flash_attn_func
-    except:
+    except Exception:
         _installed = False
         if version.parse(torch.__version__) < version.parse("2.0.0"):
             _installed = True
             try:
                 from flash_attn.flash_attn_triton import flash_attn_func
-            except:
+            except Exception:
                 _installed = False
         if not _installed:
             raise RuntimeError(
@@ -292,9 +291,9 @@ def triton_flash_attn_fn(
         _s_k = max(0, attn_bias.size(3) - key.size(1))
         attn_bias = attn_bias[:, :, _s_q:, _s_k:]
     if dropout_p:
-        raise NotImplementedError(f"Dropout not implemented for attn_impl: triton.")
+        raise NotImplementedError("Dropout not implemented for attn_impl: triton.")
     if needs_weights:
-        raise NotImplementedError(f"attn_impl: triton cannot return attn weights.")
+        raise NotImplementedError("attn_impl: triton cannot return attn weights.")
     if key_padding_mask is not None:
         warnings.warn(
             "Propagating key_padding_mask to the attention module "
@@ -337,17 +336,17 @@ class MultiheadAttention(nn.Module):
         weights,
     ):
         super().__init__()
-        attn_impl = config.attn_config["attn_impl"]
-        self.attn_impl = config.attn_config["attn_impl"]
-        self.clip_qkv = config.attn_config["clip_qkv"]
-        self.qk_ln = config.attn_config["qk_ln"]
+        attn_impl = config.attn_config.attn_impl
+        self.attn_impl = config.attn_config.attn_impl
+        self.clip_qkv = config.attn_config.clip_qkv
+        self.qk_ln = config.attn_config.qk_ln
         self.d_model = config.d_model
         d_model = config.d_model
         self.n_heads = config.n_heads
-        self.softmax_scale = config.attn_config["softmax_scale"]
+        self.softmax_scale = config.attn_config.softmax_scale
         if self.softmax_scale is None:
             self.softmax_scale = 1 / math.sqrt(self.d_model / self.n_heads)
-        self.attn_dropout_p = config.attn_config["attn_pdrop"]
+        self.attn_dropout_p = config.attn_config.attn_pdrop
 
         if self.n_heads % weights.process_group.size() != 0:
             raise ValueError(
@@ -428,24 +427,24 @@ class MultiQueryAttention(nn.Module):
     additive bias.
     """
 
-    def __init__(self, config, prefix, weights):
+    def __init__(self, config, prefix, weights, verbose=False):
         super().__init__()
-        attn_impl = config.attn_config["attn_impl"]
-        self.attn_impl = config.attn_config["attn_impl"]
-        self.clip_qkv = config.attn_config["clip_qkv"]
-        self.qk_ln = config.attn_config["qk_ln"]
+        attn_impl = config.attn_config.attn_impl
+        self.attn_impl = config.attn_config.attn_impl
+        self.clip_qkv = config.attn_config.clip_qkv
+        self.qk_ln = config.attn_config.qk_ln
         self.d_model = config.d_model
         d_model = config.d_model
         self.n_heads = config.n_heads
-        self.softmax_scale = config.attn_config["softmax_scale"]
+        self.softmax_scale = config.attn_config.softmax_scale
         if self.softmax_scale is None:
             self.softmax_scale = 1 / math.sqrt(self.head_dim)
-        self.attn_dropout_p = config.attn_config["attn_pdrop"]
+        self.attn_dropout_p = config.attn_config.attn_pdrop
         # self.Wqkv = nn.Linear(d_model, d_model + 2 * self.head_dim, device=device)
         self.Wqkv = TensorParallelColumnLinear.load(
             config, prefix=f"{prefix}.Wqkv", weights=weights, bias=not config.no_bias
         )
-        fuse_splits = (d_model, d_model + self.head_dim)
+        (d_model, d_model + self.head_dim)
         if self.qk_ln:
             raise NotImplementedError("qk_ln not supported")
         if self.attn_impl == "flash":
@@ -614,9 +613,9 @@ class MPTBlock(nn.Module):
     def __init__(self, config, prefix, weights):
         super().__init__()
         self.prefix = prefix
-        if config.attn_config["attn_type"] != "multihead_attention":
+        if config.attn_config.attn_type != "multihead_attention":
             raise NotImplementedError(
-                f"""Not implemented attn {config.attn_config["attn_type"]}"""
+                f"""Not implemented attn {config.attn_config.attn_type}"""
             )
         resid_pdrop = config.resid_pdrop
         if config.no_bias:
@@ -783,19 +782,21 @@ class MPTPreTrainedModel(PreTrainedModel):
 
 
 class MPTModel(MPTPreTrainedModel):
-    def __init__(self, config, weights):
+    def __init__(self, prefix: str, config, weights):
         # config._validate_config()
         super().__init__(config)
         self.world_size = weights.process_group.size()
         self.rank = weights.process_group.rank()
         self.n_heads = config.n_heads
-        self.attn_impl = config.attn_config["attn_impl"]
-        self.prefix_lm = config.attn_config["prefix_lm"]
-        self.attn_uses_sequence_id = config.attn_config["attn_uses_sequence_id"]
-        self.alibi = config.attn_config["alibi"]
-        self.alibi_bias_max = config.attn_config["alibi_bias_max"]
+        self.attn_impl = config.attn_config.attn_impl
+        self.prefix_lm = config.attn_config.prefix_lm
+        self.attn_uses_sequence_id = config.attn_config.attn_uses_sequence_id
+        self.alibi = config.attn_config.alibi
+        self.alibi_bias_max = config.attn_config.alibi_bias_max
         if config.init_device == "mixed":
-            if dist.get_local_rank() == 0:
+            # TODO: reimplement mixed device initialization
+            # dist.get_local_rank() == 0:
+            if True:
                 config.init_device = "cpu"
             else:
                 config.init_device = "meta"
@@ -809,13 +810,13 @@ class MPTModel(MPTPreTrainedModel):
                 f"Requested norm type ({config.norm_type}) is not implemented within this repo."
             )
 
-        self.wte = TensorParallelEmbedding("transformer.wte", weights)
+        self.wte = TensorParallelEmbedding(f"{prefix}.wte", weights)
 
         if not self.alibi:
-            self.wpe = TensorParallelEmbedding("transformer.wpe", weights)
+            self.wpe = TensorParallelEmbedding(f"{prefix}.wpe", weights)
         self.blocks = nn.ModuleList(
             [
-                MPTBlock(config, prefix=f"transformer.blocks.{i}", weights=weights)
+                MPTBlock(config, prefix=f"{prefix}.blocks.{i}", weights=weights)
                 for i in range(config.n_layers)
             ]
         )
@@ -1016,7 +1017,7 @@ class MPTModel(MPTPreTrainedModel):
             if past_key_values is not None:
                 if len(past_key_values) != self.config.n_layers:
                     raise ValueError(
-                        f"past_key_values must provide a past_key_value for each attention "
+                        "past_key_values must provide a past_key_value for each attention "
                         + f"layer in the network (len(past_key_values)={len(past_key_values)!r}; self.config.n_layers={self.config.n_layers!r})."
                     )
                 past_position = past_key_values[0][0].size(1)
@@ -1085,13 +1086,19 @@ class MPTModel(MPTPreTrainedModel):
 
 
 class MPTForCausalLM(MPTPreTrainedModel):
-    def __init__(self, config, weights):
+    def __init__(self, prefix: str, config, weights):
         super().__init__(config)
+
+        if not prefix:
+            prefix = "transformer"
+        else:
+            prefix = f"{prefix}.transformer"
+
         if not config.tie_word_embeddings:
             raise ValueError("MPTForCausalLM only supports tied word embeddings")
-        self.transformer = MPTModel(config, weights)
+        self.transformer = MPTModel(prefix, config, weights)
         self.lm_head = SpeculativeHead.load(
-            config, prefix="transformer.wte", weights=weights
+            config, prefix=f"{prefix}.wte", weights=weights
         )
         self.logit_scale = None
         if config.logit_scale is not None:
@@ -1176,7 +1183,7 @@ class MPTForCausalLM(MPTPreTrainedModel):
             input_ids = input_ids[:, -1].unsqueeze(-1)
         if self.transformer.prefix_lm:
             prefix_mask = torch.ones_like(attention_mask)
-            if kwargs.get("use_cache") == False:
+            if kwargs.get("use_cache") is False:
                 raise NotImplementedError(
                     "MPT with prefix_lm=True does not support use_cache=False."
                 )
diff --git a/server/text_generation_server/models/custom_modeling/neox_modeling.py b/server/text_generation_server/models/custom_modeling/neox_modeling.py
index fcad32fa..06731a6f 100644
--- a/server/text_generation_server/models/custom_modeling/neox_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/neox_modeling.py
@@ -21,25 +21,14 @@ import torch
 import torch.distributed
 import torch.utils.checkpoint
 from torch import nn
-from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss
+from torch.nn import CrossEntropyLoss
 
 from transformers.activations import ACT2FN
-from transformers.file_utils import (
-    add_code_sample_docstrings,
-    add_start_docstrings,
-    add_start_docstrings_to_model_forward,
-    replace_return_docstrings,
-)
 from transformers.modeling_outputs import (
     BaseModelOutputWithPast,
     CausalLMOutputWithPast,
-    QuestionAnsweringModelOutput,
-    SequenceClassifierOutputWithPast,
-    TokenClassifierOutput,
 )
 from transformers.modeling_utils import PreTrainedModel
-from transformers import GPTNeoXConfig
-from loguru import logger
 from text_generation_server.layers import (
     TensorParallelColumnLinear,
     TensorParallelEmbedding,
@@ -133,7 +122,6 @@ class GPTNeoXAttention(nn.Module):
         self.hidden_size = config.hidden_size
         self.head_size = self.hidden_size // self.num_attention_heads
         self.rotary_ndims = int(self.head_size * config.rotary_pct)
-        max_positions = config.max_position_embeddings
         # ??? TODO
         # self.register_buffer(
         #     "bias",
@@ -404,24 +392,24 @@ class GPTNeoXMLP(nn.Module):
 
 
 class GPTNeoXLayer(nn.Module):
-    def __init__(self, layer_id, config, weights):
+    def __init__(self, layer_id, prefix: str, config, weights):
         super().__init__()
         self.use_parallel_residual = config.use_parallel_residual
         self.input_layernorm = nn.LayerNorm.load(
-            prefix=f"gpt_neox.layers.{layer_id}.input_layernorm",
+            prefix=f"{prefix}.layers.{layer_id}.input_layernorm",
             weights=weights,
             eps=config.layer_norm_eps,
         )
         self.post_attention_layernorm = nn.LayerNorm.load(
-            prefix=f"gpt_neox.layers.{layer_id}.post_attention_layernorm",
+            prefix=f"{prefix}.layers.{layer_id}.post_attention_layernorm",
             weights=weights,
             eps=config.layer_norm_eps,
         )
         self.attention = GPTNeoXAttention(
-            config, prefix=f"gpt_neox.layers.{layer_id}.attention", weights=weights
+            config, prefix=f"{prefix}.layers.{layer_id}.attention", weights=weights
         )
         self.mlp = GPTNeoXMLP(
-            config, prefix=f"gpt_neox.layers.{layer_id}.mlp", weights=weights
+            config, prefix=f"{prefix}.layers.{layer_id}.mlp", weights=weights
         )
 
     def forward(
@@ -472,23 +460,23 @@ class GPTNeoXLayer(nn.Module):
 
 
 class GPTNeoXModel(GPTNeoXPreTrainedModel):
-    def __init__(self, config, weights):
+    def __init__(self, prefix: str, config, weights):
         super().__init__(config)
         self.config = config
 
         self.num_attention_heads = config.num_attention_heads
 
         self.embed_in = TensorParallelEmbedding(
-            prefix="gpt_neox.embed_in", weights=weights
+            prefix=f"{prefix}.embed_in", weights=weights
         )
         self.layers = nn.ModuleList(
             [
-                GPTNeoXLayer(layer_id, config, weights)
+                GPTNeoXLayer(layer_id, prefix, config, weights)
                 for layer_id in range(config.num_hidden_layers)
             ]
         )
         self.final_layer_norm = nn.LayerNorm.load(
-            prefix="gpt_neox.final_layer_norm",
+            prefix=f"{prefix}.final_layer_norm",
             weights=weights,
             eps=config.layer_norm_eps,
         )
@@ -640,9 +628,15 @@ class GPTNeoXModel(GPTNeoXPreTrainedModel):
 class GPTNeoxForCausalLM(GPTNeoXPreTrainedModel):
     _keys_to_ignore_on_load_missing = [r"position_ids", r"predictions.decoder.bias"]
 
-    def __init__(self, config, weights):
+    def __init__(self, prefix: str, config, weights):
         super().__init__(config)
-        self.gpt_neox = GPTNeoXModel(config, weights)
+
+        if not prefix:
+            prefix = "gpt_neox"
+        else:
+            prefix = f"{prefix}.gpt_neox"
+
+        self.gpt_neox = GPTNeoXModel(prefix, config, weights)
         self.embed_out = SpeculativeHead.load(
             config, prefix="embed_out", weights=weights
         )
diff --git a/server/text_generation_server/models/custom_modeling/opt_modeling.py b/server/text_generation_server/models/custom_modeling/opt_modeling.py
index 83d62dea..bd440321 100644
--- a/server/text_generation_server/models/custom_modeling/opt_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/opt_modeling.py
@@ -94,11 +94,13 @@ class OPTLearnedPositionalEmbedding(nn.Module):
     This module learns positional embeddings up to a fixed maximum size.
     """
 
-    def __init__(self, weights):
+    def __init__(self, prefix: str, weights):
         super().__init__()
         self.offset = 2
         self.weight = nn.Parameter(
-            weights.get_tensor("model.decoder.embed_positions.weight")
+            weights.get_tensor(
+                f"{prefix + '.' if prefix else ''}decoder.embed_positions.weight"
+            )
         )
 
     def forward(
@@ -311,11 +313,11 @@ class OPTAttention(nn.Module):
 
 
 class OPTDecoderLayer(nn.Module):
-    def __init__(self, layer_id: int, config: OPTConfig, weights):
+    def __init__(self, layer_id: int, prefix: str, config: OPTConfig, weights):
         super().__init__()
         self.process_group = weights.process_group
         self.hidden_size = config.hidden_size
-        prefix = f"model.decoder.layers.{layer_id}"
+        prefix = f"{prefix + '.' if prefix else ''}decoder.layers.{layer_id}"
         self.self_attn = OPTAttention(
             config,
             prefix=f"{prefix}.self_attn",
@@ -429,7 +431,7 @@ class OPTPreTrainedModel(PreTrainedModel):
 
 
 class OPTDecoder(OPTPreTrainedModel):
-    def __init__(self, config: OPTConfig, weights):
+    def __init__(self, prefix: str, config: OPTConfig, weights):
         super().__init__(config)
         self.dropout = config.dropout
         self.layerdrop = config.layerdrop
@@ -437,21 +439,29 @@ class OPTDecoder(OPTPreTrainedModel):
         self.max_target_positions = config.max_position_embeddings
         self.vocab_size = config.vocab_size
 
+        prefix = prefix + "." if prefix else ""
+
         self.embed_tokens = TensorParallelEmbedding(
-            prefix="model.decoder.embed_tokens", weights=weights
+            prefix=f"{prefix}decoder.embed_tokens", weights=weights
         )
-        self.embed_positions = OPTLearnedPositionalEmbedding(weights)
+        self.embed_positions = OPTLearnedPositionalEmbedding(prefix, weights)
 
         if config.word_embed_proj_dim != config.hidden_size:
             self.project_out = FastLinear.load(
-                config, prefix="model.decoder.project_out", weights=weights, bias=False
+                config,
+                prefix=f"{prefix}decoder.project_out",
+                weights=weights,
+                bias=False,
             )
         else:
             self.project_out = None
 
         if config.word_embed_proj_dim != config.hidden_size:
             self.project_in = FastLinear.load(
-                config, prefix="model.decoder.project_in", weights=weights, bias=False
+                config,
+                prefix=f"{prefix}decoder.project_in",
+                weights=weights,
+                bias=False,
             )
         else:
             self.project_in = None
@@ -461,14 +471,14 @@ class OPTDecoder(OPTPreTrainedModel):
         # see https://github.com/facebookresearch/metaseq/pull/164
         if config.do_layer_norm_before and not config._remove_final_layer_norm:
             self.final_layer_norm = nn.LayerNorm.load(
-                prefix="model.decoder.final_layer_norm", weights=weights, eps=EPS
+                prefix=f"{prefix}decoder.final_layer_norm", weights=weights, eps=EPS
             )
         else:
             self.final_layer_norm = None
 
         self.layers = nn.ModuleList(
             [
-                OPTDecoderLayer(layer_id, config, weights)
+                OPTDecoderLayer(layer_id, prefix, config, weights)
                 for layer_id in range(config.num_hidden_layers)
             ]
         )
@@ -686,9 +696,9 @@ class OPTDecoder(OPTPreTrainedModel):
 
 
 class OPTModel(OPTPreTrainedModel):
-    def __init__(self, config: OPTConfig, weights):
+    def __init__(self, prefix: str, config: OPTConfig, weights):
         super().__init__(config)
-        self.decoder = OPTDecoder(config, weights)
+        self.decoder = OPTDecoder(prefix, config, weights)
         # Initialize weights and apply final processing
 
     def forward(
@@ -743,13 +753,15 @@ class OPTModel(OPTPreTrainedModel):
 
 
 class OPTForCausalLM(OPTPreTrainedModel):
-    def __init__(self, config, weights):
+    def __init__(self, prefix, config, weights):
         super().__init__(config)
 
-        self.model = OPTModel(config, weights)
+        self.model = OPTModel(prefix, config, weights)
 
         self.lm_head = SpeculativeHead.load(
-            config, prefix="model.decoder.embed_tokens", weights=weights
+            config,
+            prefix=f"{prefix + '.' if prefix else ''}decoder.embed_tokens",
+            weights=weights,
         )
 
     def forward(
@@ -792,7 +804,7 @@ class OPTForCausalLM(OPTPreTrainedModel):
             return_dict=return_dict,
         )
 
-        logits, speculative_logits = self.lm_head(outputs)
+        logits, speculative_logits = self.lm_head(outputs.last_hidden_state)
 
         loss = None
 
diff --git a/server/text_generation_server/models/custom_modeling/phi_modeling.py b/server/text_generation_server/models/custom_modeling/phi_modeling.py
index 04b470eb..3f2ed010 100644
--- a/server/text_generation_server/models/custom_modeling/phi_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/phi_modeling.py
@@ -5,7 +5,7 @@ import torch.distributed
 
 import math
 from torch import nn
-from typing import Optional, List, Tuple, Any
+from typing import Optional, List, Tuple
 from transformers.configuration_utils import PretrainedConfig
 from transformers.modeling_outputs import CausalLMOutputWithPast
 
@@ -248,16 +248,16 @@ class PhiBlock(nn.Module):
 
 # PhiModel implements the embedding layer and the transformer blocks.
 class PhiModel(nn.Module):
-    def __init__(self, config, weights):
+    def __init__(self, prefix: str, config, weights):
         super().__init__()
         self.tp_rank = weights.process_group.rank()
         self.tp_world_size = weights.process_group.size()
         self.embed_tokens = TensorParallelEmbedding(
-            prefix="transformer.embd.wte", weights=weights
+            prefix=f"{prefix}.embd.wte", weights=weights
         )
         self.blocks = nn.ModuleList(
             [
-                PhiBlock(f"transformer.h.{layer_id}", config, weights)
+                PhiBlock(f"{prefix}.h.{layer_id}", config, weights)
                 for layer_id in range(config.n_layer)
             ]
         )
@@ -289,9 +289,15 @@ class PhiModel(nn.Module):
 
 # PhiForCausalLM wraps the PhiModel and PhiCausalLMHead together and returns a CausalLMOutputWithPast object.
 class PhiForCausalLM(torch.nn.Module):
-    def __init__(self, config, weights):
+    def __init__(self, prefix: str, config, weights):
         super().__init__()
-        self.model = PhiModel(config, weights)
+
+        if not prefix:
+            prefix = "transformer"
+        else:
+            prefix = f"{prefix}.transformer"
+
+        self.model = PhiModel(prefix, config, weights)
         self.lm_head = PhiCausalLMHead(config, weights)
 
     def forward(
diff --git a/server/text_generation_server/models/custom_modeling/siglip.py b/server/text_generation_server/models/custom_modeling/siglip.py
index 5fbc6d29..95ac9ede 100644
--- a/server/text_generation_server/models/custom_modeling/siglip.py
+++ b/server/text_generation_server/models/custom_modeling/siglip.py
@@ -1,20 +1,15 @@
-from typing import Optional, Tuple, Union
-
+from typing import Optional, Tuple
+import warnings
 import math
 import torch
 from torch import nn
 
 from transformers.activations import ACT2FN
-from transformers.modeling_attn_mask_utils import (
-    _create_4d_causal_attention_mask,
-    _prepare_4d_attention_mask,
-)
 from transformers.modeling_outputs import (
-    BaseModelOutput,
     BaseModelOutputWithPooling,
-    ImageClassifierOutput,
 )
-from transformers import SiglipConfig, SiglipTextConfig, SiglipVisionConfig
+from transformers import SiglipConfig, SiglipVisionConfig
+from torch.nn.init import _calculate_fan_in_and_fan_out
 
 from text_generation_server.layers.tensor_parallel import (
     TensorParallelEmbedding,
@@ -244,9 +239,6 @@ class SiglipMultiheadAttentionPoolingHead(nn.Module):
         return hidden_state[:, 0]
 
 
-import warnings
-
-
 def _trunc_normal_(tensor, mean, std, a, b):
     # Cut & paste from PyTorch official master until it's in a few official releases - RW
     # Method based on https://people.sc.fsu.edu/~jburkardt/presentations/truncated_normal.pdf
@@ -264,12 +256,12 @@ def _trunc_normal_(tensor, mean, std, a, b):
     # Values are generated by using a truncated uniform distribution and
     # then using the inverse CDF for the normal distribution.
     # Get upper and lower cdf values
-    l = norm_cdf((a - mean) / std)
-    u = norm_cdf((b - mean) / std)
+    lower = norm_cdf((a - mean) / std)
+    upper = norm_cdf((b - mean) / std)
 
     # Uniformly fill tensor with values from [l, u], then translate to
     # [2l-1, 2u-1].
-    tensor.uniform_(2 * l - 1, 2 * u - 1)
+    tensor.uniform_(2 * lower - 1, 2 * upper - 1)
 
     # Use inverse cdf transform for normal distribution to get truncated
     # standard normal
@@ -313,9 +305,6 @@ def trunc_normal_tf_(
         tensor.mul_(std).add_(mean)
 
 
-from torch.nn.init import _calculate_fan_in_and_fan_out
-
-
 def variance_scaling_(tensor, scale=1.0, mode="fan_in", distribution="normal"):
     fan_in, fan_out = _calculate_fan_in_and_fan_out(tensor)
     if mode == "fan_in":
@@ -349,9 +338,6 @@ def default_flax_embed_init(tensor):
     variance_scaling_(tensor, mode="fan_in", distribution="normal")
 
 
-from transformers import PreTrainedModel
-
-
 class SiglipEncoder(nn.Module):
     """
     Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
@@ -378,7 +364,6 @@ class SiglipEncoder(nn.Module):
         inputs_embeds,
         attention_mask: Optional[torch.Tensor] = None,
     ):
-
         hidden_states = inputs_embeds
         for idx, encoder_layer in enumerate(self.layers):
             hidden_states, _ = encoder_layer(
@@ -393,7 +378,6 @@ class SiglipVisionTransformer(nn.Module):
     def __init__(self, prefix, config: SiglipVisionConfig, weights):
         super().__init__()
         self.config = config
-        embed_dim = config.hidden_size
 
         self.embeddings = SiglipVisionEmbeddings(
             prefix=f"{prefix}.embeddings", config=config, weights=weights
@@ -401,20 +385,11 @@ class SiglipVisionTransformer(nn.Module):
         self.encoder = SiglipEncoder(
             prefix=f"{prefix}.encoder", config=config, weights=weights
         )
-        self.post_layernorm = nn.LayerNorm.load(
-            prefix=f"{prefix}.post_layernorm",
-            weights=weights,
-            eps=config.layer_norm_eps,
-        )
 
     def forward(
         self,
         pixel_values: Optional[torch.FloatTensor] = None,
     ):
-        r"""
-        Returns:
-
-        """
         if pixel_values is None:
             raise ValueError("You have to specify pixel_values")
 
@@ -427,10 +402,9 @@ class SiglipVisionTransformer(nn.Module):
             inputs_embeds=hidden_states,
         )
         last_hidden_state = encoder_outputs
-        post_last_hidden_state = self.post_layernorm(last_hidden_state)
 
         return BaseModelOutputWithPooling(
-            last_hidden_state=post_last_hidden_state,
+            last_hidden_state=last_hidden_state,
             # pooler_output=pooled_output,
             # hidden_states=encoder_outputs,
         )
diff --git a/server/text_generation_server/models/custom_modeling/t5_modeling.py b/server/text_generation_server/models/custom_modeling/t5_modeling.py
index 0b899fba..e6666acd 100644
--- a/server/text_generation_server/models/custom_modeling/t5_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/t5_modeling.py
@@ -45,6 +45,15 @@ from text_generation_server.layers import (
     SpeculativeHead,
 )
 
+# copied from https://github.com/huggingface/transformers/blob/cd4584e3c809bb9e1392ccd3fe38b40daba5519a/src/transformers/models/t5/modeling_t5.py#L1316
+# Warning message for FutureWarning: head_mask was separated into two input args - head_mask, decoder_head_mask
+__HEAD_MASK_WARNING_MSG = """
+The input argument `head_mask` was split into two arguments `head_mask` and `decoder_head_mask`. Currently,
+`decoder_head_mask` is set to copy `head_mask`, but this feature is deprecated and will be removed in future versions.
+If you do not want to use any `decoder_head_mask` now, please set `decoder_head_mask = torch.ones(num_layers,
+num_heads)`.
+"""
+
 
 class PartialTPEmbedding(nn.Module):
     def __init__(self, prefix: str, weights):
@@ -1132,12 +1141,12 @@ class T5ForConditionalGeneration(T5PreTrainedModel):
         if labels is not None:
             loss_fct = CrossEntropyLoss(ignore_index=-100)
             # move labels to correct device to enable PP
-            labels = labels.to(lm_logits.device)
-            loss = loss_fct(lm_logits.view(-1, lm_logits.size(-1)), labels.view(-1))
+            labels = labels.to(logits.device)
+            loss = loss_fct(logits.view(-1, logits.size(-1)), labels.view(-1))
             # TODO(thom): Add z_loss https://github.com/tensorflow/mesh/blob/fa19d69eafc9a482aff0b59ddd96b025c0cb207d/mesh_tensorflow/layers.py#L666
 
         if not return_dict:
-            output = (lm_logits,) + decoder_outputs[1:] + encoder_outputs
+            output = (logits,) + decoder_outputs[1:] + encoder_outputs
             return ((loss,) + output) if loss is not None else output
 
         return (
diff --git a/server/text_generation_server/models/custom_modeling/vlm.py b/server/text_generation_server/models/custom_modeling/vlm.py
index b74b43ff..e5c44045 100644
--- a/server/text_generation_server/models/custom_modeling/vlm.py
+++ b/server/text_generation_server/models/custom_modeling/vlm.py
@@ -42,7 +42,7 @@ def load_vision_model(prefix, config, weights):
         )
 
         return SiglipVisionTransformer(
-            prefix=f"vision_tower.vision_model", config=config, weights=weights
+            prefix="vision_tower.vision_model", config=config, weights=weights
         )
     else:
         raise RuntimeError(f"Unsupported model type {config.model_type}")
diff --git a/server/text_generation_server/models/flash_causal_lm.py b/server/text_generation_server/models/flash_causal_lm.py
index 86d9b4c8..bc9d44a0 100644
--- a/server/text_generation_server/models/flash_causal_lm.py
+++ b/server/text_generation_server/models/flash_causal_lm.py
@@ -1,7 +1,7 @@
+from contextlib import nullcontext
 import math
 import os
 import time
-import itertools
 import torch
 import torch.distributed
 
@@ -10,31 +10,47 @@ import numpy as np
 from loguru import logger
 from dataclasses import dataclass
 from opentelemetry import trace
-from transformers import PreTrainedTokenizerBase
-from typing import Optional, Tuple, List, Type, Dict
+from transformers import (
+    PreTrainedTokenizerBase,
+    AutoConfig,
+    AutoTokenizer,
+    GenerationConfig,
+)
+from typing import Any, ContextManager, Iterable, Optional, Tuple, List, Type, Dict
 
+from text_generation_server.adapters import AdapterBatchData, AdapterBatchMetadata
 from huggingface_hub.constants import HUGGINGFACE_HUB_CACHE
+from text_generation_server.utils.chunks import concat_text_chunks
 from text_generation_server.utils.import_utils import SYSTEM
 from text_generation_server.models import Model
+from text_generation_server.utils.log import log_master
 from text_generation_server.utils.tokens import batch_top_tokens
-from text_generation_server.utils.dist import RANK
 from text_generation_server.utils.speculate import get_speculate
+from text_generation_server.utils import (
+    initialize_torch_distributed,
+    weight_files,
+    Weights,
+)
 from text_generation_server.models.types import (
     Batch,
     Tokens,
     Generation,
     GeneratedText,
 )
-from text_generation_server.models.cache_manager import (
-    get_cache_manager,
-    set_cache_manager,
-    BLOCK_SIZE,
-)
 from text_generation_server.pb import generate_pb2
-from text_generation_server.models.globals import MEM_POOL, CUDA_GRAPHS
-import text_generation_server.models.globals as tgi_globals
+from text_generation_server.models.globals import (
+    MEM_POOL,
+    ATTENTION,
+    BLOCK_SIZE,
+    CUDA_GRAPHS,
+    TGI_WIGGLE_ROOM,
+    get_adapter_to_index,
+)
+from text_generation_server.layers.attention import Seqlen
 from text_generation_server.utils import StoppingCriteria, HeterogeneousNextTokenChooser
 from text_generation_server.utils.dist import MEMORY_FRACTION
+from text_generation_server.utils.quantization import get_loader
+from text_generation_server.utils.segments import SegmentConcatBuilder, find_segments
 
 from text_generation_server.utils.import_utils import (
     empty_cache,
@@ -45,6 +61,54 @@ from text_generation_server.utils.import_utils import (
 tracer = trace.get_tracer(__name__)
 
 
+# Will be set in init
+SLIDING_WINDOW: Optional[int] = None
+
+
+def set_sliding_window(sliding_window: int):
+    global SLIDING_WINDOW
+    SLIDING_WINDOW = sliding_window
+
+
+def get_sliding_windows() -> int:
+    global SLIDING_WINDOW
+    return SLIDING_WINDOW
+
+
+def init_cpu_threads_env(rank_id: int, world_size: int):
+    import importlib.util
+
+    if importlib.util.find_spec("numa") is not None:
+        import numa
+        import psutil
+
+        nodes = numa.info.get_max_node() + 1
+        rank_per_node = math.ceil(world_size / nodes)
+        num_cpus_per_nodes = int(psutil.cpu_count(logical=False) / nodes)
+        node_id = int(rank_id / rank_per_node)
+        rank_offset_per_node = rank_id % rank_per_node
+        if os.getenv("OMP_NUM_THREADS") is None:
+            num_cpus_per_rank = max(int(num_cpus_per_nodes / rank_per_node), 1)
+        else:
+            num_cpus_per_rank = int(os.getenv("OMP_NUM_THREADS"))
+        if len(numa.memory.get_membind_nodes()) == nodes:
+            numa.memory.set_membind_nodes((node_id))
+        torch.set_num_threads(num_cpus_per_rank)
+        if len(numa.schedule.get_affinitive_cpus(0)) == psutil.cpu_count(logical=True):
+            cpu_start = num_cpus_per_rank * rank_offset_per_node
+            numa.schedule.run_on_cpus(
+                0,
+                *(
+                    numa.info.node_to_cpus(node_id)[
+                        cpu_start : cpu_start + num_cpus_per_rank
+                    ]
+                ),
+            )
+        logger.info(
+            f"affinity={numa.schedule.get_affinitive_cpus(0)}, membind = {numa.memory.get_membind_nodes()}"
+        )
+
+
 @dataclass
 class FlashCausalLMBatch(Batch):
     batch_id: int
@@ -55,12 +119,15 @@ class FlashCausalLMBatch(Batch):
     # Decoder values
     input_ids: torch.Tensor
     position_ids: torch.Tensor
-    speculative_ids: torch.Tensor
+    speculative_ids: Optional[torch.Tensor]
 
     # Flash Attention values
 
     # tensor of length b containing the cumulative sequence lengths of the sequences in the batch, only used in prefill
     cu_seqlen_prefill: Optional[torch.Tensor]
+    # Prefill cache indices is used to slice into the kv tensor before caching it into the paged attention buffers
+    # as we only keep SLIDING_WINDOW values instead of the whole tensor
+    prefill_cache_indices: Optional[torch.Tensor]
 
     # Paged Attention values
 
@@ -69,16 +136,16 @@ class FlashCausalLMBatch(Batch):
     start_slots: torch.Tensor
     # tensor of indices of the currently used slots, length = \sum_{i=0}^{b} s_i in prefill, length = b in decode
     slot_indices: torch.Tensor
-    # List of tuple of ints representing the number of blocks and slots needed by each sequence
-    needed_blocks_slots: Optional[List[Tuple[int, int]]]
 
-    # Set in prefill by the CacheManager
     # list of length b of list of length s_i // block_size
-    block_tables: Optional[List[List[int]]]
+    block_tables: List[List[int]]
     # tensor of size [b, max_total_seqlen // block_size] holding the paged attention block tables for all sequences
-    block_tables_tensor: Optional[torch.Tensor]
+    block_tables_tensor: torch.Tensor
     # tensor of length \sum_{i=0}^{b} max_s_i  holding the paged attention slots for all sequences
-    slots: Optional[torch.Tensor]
+    slots: torch.Tensor
+    # size [b], containing the number of blocks that can be retrieved from the cache
+    prefix_lens: List[int]
+    prefix_lens_tensor: torch.Tensor
 
     max_seqlen: int
 
@@ -87,6 +154,9 @@ class FlashCausalLMBatch(Batch):
     prefill_next_token_indices: Optional[torch.tensor]
     prefill_cu_outlens: Optional[List[int]]
 
+    # Prefixes
+    prefix_ids: List[List[int]]
+
     # All tokens
     all_input_ids: List[List[int]]
     all_input_ids_tensor: torch.Tensor
@@ -103,8 +173,11 @@ class FlashCausalLMBatch(Batch):
     top_n_tokens: List[int]
     top_n_tokens_tensor: torch.Tensor
 
+    # Adapter metadata for each request
+    adapter_meta: AdapterBatchMetadata
+
     # Number of blocks in this batch
-    blocks: int
+    num_blocks: int
     # Maximum number of blocks
     max_blocks: int
 
@@ -113,32 +186,28 @@ class FlashCausalLMBatch(Batch):
             id=self.batch_id,
             request_ids=[r.id for r in self.requests],
             size=len(self),
-            max_tokens=self.blocks * BLOCK_SIZE,
+            max_tokens=self.num_blocks * BLOCK_SIZE,
         )
 
     @classmethod
-    def batch_tokenized_inputs(cls, requests, tokenizer):
-        batch_inputs = []
-        max_truncation = 0
+    def batch_tokenized_inputs(
+        cls, requests: Iterable[generate_pb2.Request], tokenizer
+    ):
+        max_length = 0
+        all_input_ids = []
+        batch_size = 0
         for r in requests:
-            batch_inputs.append(r.inputs)
-            max_truncation = max(max_truncation, r.truncate)
-
-        batch_tokenized_inputs = tokenizer(
-            batch_inputs, truncation=True, max_length=max_truncation
-        )["input_ids"]
-        return batch_tokenized_inputs
-
-    @classmethod
-    def from_pb(
-        cls,
-        pb: generate_pb2.Batch,
-        tokenizer: PreTrainedTokenizerBase,
-        dtype: torch.dtype,
-        device: torch.device,
-    ) -> "FlashCausalLMBatch":
-        batch_tokenized_inputs = cls.batch_tokenized_inputs(pb.requests, tokenizer)
-        return cls.from_tokenized(pb, tokenizer, batch_tokenized_inputs, dtype, device)
+            batch_size += 1
+            inputs = concat_text_chunks(r.input_chunks.chunks)
+            input_ids = tokenizer(
+                inputs,
+                truncation=True,
+                max_length=r.truncate,
+                add_special_tokens=r.add_special_tokens,
+            )["input_ids"]
+            max_length = max(max_length, len(input_ids))
+            all_input_ids.append(input_ids)
+        return all_input_ids
 
     @classmethod
     def from_tokenized(
@@ -149,17 +218,18 @@ class FlashCausalLMBatch(Batch):
         dtype: torch.dtype,
         device: torch.device,
     ) -> "FlashCausalLMBatch":
+        sliding_window = get_sliding_windows()
         position_ids = []
-        speculative_ids = []
         cu_seqlen_prefill = [0]
-        needed_blocks_slots = []
         start_slots = []
         slot_indices = []
+        prefill_cache_indices = []
 
         input_lengths = []
         prefix_offsets = []
         read_offsets = []
         all_input_ids = []
+        prefix_ids = []
         requests_idx_mapping = {}
 
         all_prefill_logprobs = True
@@ -172,16 +242,23 @@ class FlashCausalLMBatch(Batch):
         stopping_criterias = []
         top_n_tokens = []
 
+        adapter_indices_list = []
+        adapter_set = set()
+
         # Cumulative length
         cumulative_length = 0
-        cumulative_max_length = 0
+        cumulative_slot_tokens = 0
         prefill_out_cumulative_length = 0
 
-        blocks = 0
+        num_blocks = 0
         max_seqlen = 0
         max_length = 0
         max_blocks = 0
 
+        block_tables = []
+        slots = []
+        prefix_lens = []
+
         # Parse batch
         for i, (r, tokenized_input) in enumerate(
             zip(pb.requests, batch_tokenized_inputs)
@@ -189,12 +266,20 @@ class FlashCausalLMBatch(Batch):
             # request id -> idx in list mapping
             requests_idx_mapping[r.id] = i
 
-            tokenized_input = tokenized_input[-r.truncate :]
-            if (
-                tokenized_input[0] == tokenizer.bos_token_id
-                and tokenized_input[1] == tokenizer.bos_token_id
-            ):
-                tokenized_input = tokenized_input[1:]
+            orig_input_length = len(tokenized_input)
+
+            prefix_len = r.prefix_len
+            assert (
+                prefix_len <= orig_input_length
+            ), f"Prefix {prefix_len} vs input {orig_input_length}"
+            if prefix_len == orig_input_length:
+                assert prefix_len > 0
+                prefix_len -= 1
+
+            # Commented as it's costly.
+            # log_master(logger.debug, "Tokenized input ids {tokenized_input}")
+            prefix_ids.append(tokenized_input[:prefix_len])
+            tokenized_input = tokenized_input[prefix_len:]
 
             input_length = len(tokenized_input)
             input_lengths.append(input_length)
@@ -205,7 +290,9 @@ class FlashCausalLMBatch(Batch):
             all_input_ids.append(tokenized_input)
 
             # Position ids
-            request_position_ids = torch.arange(0, input_length, dtype=torch.int32)
+            request_position_ids = torch.arange(
+                prefix_len, orig_input_length, dtype=torch.int32
+            )
             position_ids.append(request_position_ids)
 
             # Add cumulative lengths of all previous inputs
@@ -220,23 +307,63 @@ class FlashCausalLMBatch(Batch):
             stopping_criterias.append(stopping_criteria)
             top_n_tokens.append(r.top_n_tokens)
 
+            ADAPTER_TO_INDEX = get_adapter_to_index()
+            adapter_index = ADAPTER_TO_INDEX.get(r.adapter_id, 0)
+            adapter_indices_list.append(torch.full((input_length,), adapter_index))
+            adapter_set.add(adapter_index)
+
             # Paged attention
             # Remove one as the first token des not have a past
             speculative_length = get_speculate()
             speculative_length = 0 if speculative_length is None else speculative_length
-            total_tokens = input_length + max_new_tokens - 1 + speculative_length
-            needed_blocks = math.ceil(total_tokens / BLOCK_SIZE)
-            blocks += needed_blocks
-            needed_blocks_slots.append((needed_blocks, total_tokens))
-            start_slots.append(cumulative_max_length)
+
+            # Tokens that need to be mapped to blocks.
+            block_tokens = orig_input_length + max_new_tokens - 1 + speculative_length
+
+            # Tokens that need to be mapped to slots. We don't need slots for the
+            # cached prefix (if present).
+            slot_tokens = input_length + max_new_tokens - 1 + speculative_length
+
+            # blocks and slots can be empty (for example in warmup)
+            if not r.blocks:
+                needed_blocks = math.ceil(block_tokens / BLOCK_SIZE)
+                request_blocks = [
+                    b for b in range(num_blocks, num_blocks + needed_blocks)
+                ]
+                request_slots = [
+                    s
+                    for b in request_blocks
+                    for s in range(b * BLOCK_SIZE, (b + 1) * BLOCK_SIZE)
+                ]
+            else:
+                request_blocks = r.blocks
+                request_slots = r.slots[
+                    prefix_len:  #: orig_input_length + max_new_tokens + speculative_length
+                ]
+
+            block_tables.append(request_blocks)
+
+            slots.extend(request_slots)
+            prefix_lens.append(prefix_len)
+            num_blocks += len(request_blocks)
+            start_slots.append(cumulative_slot_tokens)
 
             request_slot_indices = torch.arange(
-                cumulative_max_length,
-                cumulative_max_length + input_length,
+                cumulative_slot_tokens,
+                cumulative_slot_tokens + input_length,
                 dtype=torch.int64,
             )
             slot_indices.append(request_slot_indices)
 
+            # Create tensor to slice into the kv tensor in prefill
+            if sliding_window is not None:
+                request_prefill_cache_indices = torch.arange(
+                    cumulative_length + max(0, input_length - sliding_window),
+                    cumulative_length + input_length,
+                    dtype=torch.int64,
+                )
+                prefill_cache_indices.append(request_prefill_cache_indices)
+
             all_prefill_logprobs = all_prefill_logprobs and r.prefill_logprobs
             no_prefill_logprobs = no_prefill_logprobs and not r.prefill_logprobs
 
@@ -259,13 +386,17 @@ class FlashCausalLMBatch(Batch):
 
             # Update
             cumulative_length += input_length
-            cumulative_max_length += total_tokens
+            cumulative_slot_tokens += slot_tokens
             max_seqlen = max(max_seqlen, input_length)
-            max_blocks = max(max_blocks, needed_blocks)
+            max_blocks = max(max_blocks, len(request_blocks))
             max_length = max(
                 max_length, input_length + max_new_tokens + speculative_length
             )
 
+        adapter_indices = torch.cat(adapter_indices_list).to(
+            dtype=torch.int64, device=device
+        )
+
         next_token_chooser = HeterogeneousNextTokenChooser.from_pb(
             next_token_chooser_parameters, dtype, device, tokenizer
         )
@@ -287,21 +418,33 @@ class FlashCausalLMBatch(Batch):
             input_ids = np.concatenate(all_input_ids, dtype=np.int64)
             position_ids = torch.cat(position_ids)
             slot_indices = torch.cat(slot_indices)
+            if sliding_window is not None:
+                prefill_cache_indices = torch.cat(prefill_cache_indices)
         else:
             input_ids = all_input_ids[0]
             position_ids = position_ids[0]
             slot_indices = slot_indices[0]
+            if sliding_window is not None:
+                prefill_cache_indices = prefill_cache_indices[0]
 
         cu_seqlen_prefill = torch.tensor(
             cu_seqlen_prefill, device=device, dtype=torch.int32
         )
         position_ids = position_ids.to(device)
         slot_indices = slot_indices.to(device)
+        prefill_cache_indices = (
+            prefill_cache_indices.to(device) if sliding_window is not None else None
+        )
         input_ids = torch.tensor(input_ids, dtype=torch.int64, device=device)
         input_lengths_tensor = torch.tensor(
             input_lengths, dtype=torch.int32, device=device
         )
 
+        adapter_segments, adapter_segment_indices = find_segments(adapter_indices)
+        adapter_segments = torch.tensor(
+            adapter_segments, dtype=torch.int32, device=device
+        )
+
         if all_prefill_logprobs:
             prefill_head_indices = None
             prefill_next_token_indices = cu_seqlen_prefill[1:] - 1
@@ -319,6 +462,16 @@ class FlashCausalLMBatch(Batch):
             top_n_tokens, device=device, dtype=torch.int64
         )
 
+        slots = torch.tensor(slots, dtype=torch.int64, device=device)
+
+        block_tables_tensor = torch.zeros(
+            (len(block_tables), max_blocks), dtype=torch.int32, device="cpu"
+        )
+        for i, request_blocks in enumerate(block_tables):
+            block_tables_tensor[i, : len(request_blocks)] = torch.tensor(request_blocks)
+        block_tables_tensor = block_tables_tensor.to(device)
+        prefix_lens_tensor = torch.tensor(prefix_lens, dtype=torch.int32, device=device)
+
         return cls(
             batch_id=pb.id,
             requests=pb.requests,
@@ -326,12 +479,14 @@ class FlashCausalLMBatch(Batch):
             input_ids=input_ids,
             position_ids=position_ids,
             cu_seqlen_prefill=cu_seqlen_prefill,
+            prefill_cache_indices=prefill_cache_indices,
             start_slots=start_slots,
             slot_indices=slot_indices,
-            needed_blocks_slots=needed_blocks_slots,
-            block_tables=None,
-            block_tables_tensor=None,
-            slots=None,
+            block_tables=block_tables,
+            block_tables_tensor=block_tables_tensor,
+            slots=slots,
+            prefix_lens=prefix_lens,
+            prefix_lens_tensor=prefix_lens_tensor,
             max_seqlen=max_seqlen,
             prefill_head_indices=prefill_head_indices,
             prefill_next_token_indices=prefill_next_token_indices,
@@ -342,15 +497,34 @@ class FlashCausalLMBatch(Batch):
             read_offsets=read_offsets,
             all_input_ids=all_input_ids,
             all_input_ids_tensor=all_input_ids_tensor,
+            prefix_ids=prefix_ids,
             next_token_chooser=next_token_chooser,
             stopping_criterias=stopping_criterias,
             top_n_tokens=top_n_tokens,
             top_n_tokens_tensor=top_n_tokens_tensor,
-            blocks=blocks,
+            num_blocks=num_blocks,
             max_blocks=max_blocks,
+            adapter_meta=AdapterBatchMetadata(
+                adapter_indices=adapter_indices,
+                adapter_set=adapter_set,
+                adapter_segments=adapter_segments,
+                segment_indices=adapter_segment_indices,
+            ),
             speculative_ids=None,
         )
 
+    @classmethod
+    def from_pb(
+        cls,
+        pb: generate_pb2.Batch,
+        tokenizer: PreTrainedTokenizerBase,
+        dtype: torch.dtype,
+        device: torch.device,
+    ) -> "FlashCausalLMBatch":
+        assert len(pb.requests) > 0
+        batch_tokenized_inputs = cls.batch_tokenized_inputs(pb.requests, tokenizer)
+        return cls.from_tokenized(pb, tokenizer, batch_tokenized_inputs, dtype, device)
+
     @tracer.start_as_current_span("filter")
     def filter(self, request_ids: List[int]) -> "FlashCausalLMBatch":
         if len(request_ids) == 0:
@@ -380,15 +554,18 @@ class FlashCausalLMBatch(Batch):
         start_slots = []
         block_tables = []
         all_input_ids = []
+        prefix_ids = []
 
         input_lengths = []
+        prefix_lens = []
         prefix_offsets = []
         read_offsets = []
 
         stopping_criterias = []
         top_n_tokens = []
+        adapter_set = set()
 
-        blocks = 0
+        num_blocks = 0
         max_blocks = 0
         # Cumulative length
         cumulative_max_length = 0
@@ -402,11 +579,14 @@ class FlashCausalLMBatch(Batch):
 
             # Get length
             request_input_length = self.input_lengths[idx]
+            prefix_len = self.prefix_lens[idx]
             max_seqlen = max(max_seqlen, request_input_length)
 
             all_input_ids.append(self.all_input_ids[idx])
+            prefix_ids.append(self.prefix_ids[idx])
 
             input_lengths.append(request_input_length)
+            prefix_lens.append(prefix_len)
             prefix_offsets.append(self.prefix_offsets[idx])
             read_offsets.append(self.read_offsets[idx])
 
@@ -415,12 +595,16 @@ class FlashCausalLMBatch(Batch):
 
             top_n_tokens.append(self.top_n_tokens[idx])
 
+            ADAPTER_TO_INDEX = get_adapter_to_index()
+            adapter_index = ADAPTER_TO_INDEX.get(self.requests[idx].adapter_id, 0)
+            adapter_set.add(adapter_index)
+
             remaining_tokens = (
                 stopping_criteria.max_new_tokens - stopping_criteria.current_tokens
             )
 
             request_block_table = self.block_tables[idx]
-            blocks += len(request_block_table)
+            num_blocks += len(request_block_table)
             block_tables.append(request_block_table)
             start_slots.append(cumulative_max_length)
 
@@ -439,24 +623,15 @@ class FlashCausalLMBatch(Batch):
 
             max_blocks = max(max_blocks, len(request_block_table))
 
-        block_indices_to_free = []
-        # Iterate on all requests
-        for i, r in enumerate(self.requests):
-            # Filter requests that are not part of the new batch
-            if r.id not in requests_idx_mapping.keys():
-                block_indices_to_free.extend(self.block_tables[i])
-        # Free blocks
-        get_cache_manager().free(block_indices_to_free)
-        # Needed to avoid dropping blocks when the batches will go out of scope
-        self.block_tables = None
-
         # Index into tensors
         input_ids = self.input_ids[indices]
         position_ids = self.position_ids[indices]
+        adapter_indices = self.adapter_meta.adapter_indices[indices]
         all_input_ids_tensor = self.all_input_ids_tensor[indices]
         block_tables_tensor = self.block_tables_tensor[indices]
         input_lengths_tensor = self.input_lengths_tensor[indices]
         slots = self.slots[slot_filtering_indices]
+        prefix_lens_tensor = self.prefix_lens_tensor[indices]
         next_token_chooser = self.next_token_chooser.filter(indices)
         top_n_tokens_tensor = self.top_n_tokens_tensor[indices]
         speculative_ids = (
@@ -468,6 +643,12 @@ class FlashCausalLMBatch(Batch):
         # Move to GPU now that we have the whole tensor
         slot_indices = slot_indices.to(device)
 
+        adapter_segments, adapter_segment_indices = find_segments(adapter_indices)
+        adapter_segments = torch.tensor(
+            adapter_segments, dtype=torch.int32, device=device
+        )
+        # assert sum(len(b) for b in block_tables) == (block_tables_tensor != 0).sum()
+
         return type(self)(
             batch_id=self.batch_id,
             requests=requests,
@@ -475,9 +656,9 @@ class FlashCausalLMBatch(Batch):
             input_ids=input_ids,
             position_ids=position_ids,
             cu_seqlen_prefill=None,
+            prefill_cache_indices=None,
             start_slots=start_slots,
             slot_indices=slot_indices,
-            needed_blocks_slots=None,
             block_tables=block_tables,
             block_tables_tensor=block_tables_tensor,
             slots=slots,
@@ -487,17 +668,26 @@ class FlashCausalLMBatch(Batch):
             prefill_cu_outlens=None,
             input_lengths=input_lengths,
             input_lengths_tensor=input_lengths_tensor,
+            prefix_lens=prefix_lens,
+            prefix_lens_tensor=prefix_lens_tensor,
             prefix_offsets=prefix_offsets,
             read_offsets=read_offsets,
             all_input_ids=all_input_ids,
             all_input_ids_tensor=all_input_ids_tensor,
+            prefix_ids=prefix_ids,
             next_token_chooser=next_token_chooser,
             stopping_criterias=stopping_criterias,
             top_n_tokens=top_n_tokens,
             top_n_tokens_tensor=top_n_tokens_tensor,
-            blocks=blocks,
+            num_blocks=num_blocks,
             max_blocks=max_blocks,
             speculative_ids=speculative_ids,
+            adapter_meta=AdapterBatchMetadata(
+                adapter_indices=adapter_indices,
+                adapter_set=adapter_set,
+                adapter_segments=adapter_segments,
+                segment_indices=adapter_segment_indices,
+            ),
         )
 
     @classmethod
@@ -507,7 +697,7 @@ class FlashCausalLMBatch(Batch):
         requests = []
         requests_idx_mapping = {}
 
-        blocks = 0
+        num_blocks = 0
         total_batch_size = 0
         total_slots = 0
         max_blocks = 0
@@ -516,7 +706,7 @@ class FlashCausalLMBatch(Batch):
         for b in batches:
             total_batch_size += len(b)
             total_slots += len(b.slots)
-            blocks += b.blocks
+            num_blocks += b.num_blocks
             speculative_length = (
                 b.speculative_ids.shape[1] if b.speculative_ids is not None else 0
             )
@@ -545,16 +735,27 @@ class FlashCausalLMBatch(Batch):
         block_tables_tensor = batches[0].block_tables_tensor.new_zeros(
             (total_batch_size, max_blocks)
         )
+        prefix_lens_tensor = batches[0].prefix_lens_tensor.new_empty(total_batch_size)
         all_input_ids_tensor = batches[0].all_input_ids_tensor.new_zeros(
             (total_batch_size, max_length)
         )
         top_n_tokens_tensor = batches[0].top_n_tokens_tensor.new_zeros(
             total_batch_size,
         )
+        total_indices_size = sum(
+            b.adapter_meta.adapter_indices.shape[0] for b in batches
+        )
+        adapter_indices = batches[0].adapter_meta.adapter_indices.new_empty(
+            total_indices_size
+        )
+        adapter_set = set()
+        adapter_segment_builder = SegmentConcatBuilder()
 
         start_slots = []
         block_tables = []
+        prefix_lens = []
         all_input_ids = []
+        prefix_ids = []
 
         input_lengths = []
         prefix_offsets = []
@@ -568,6 +769,7 @@ class FlashCausalLMBatch(Batch):
         # Cumulative length
         cumulative_batch_size = 0
         cumulative_slots = 0
+        cumulative_adapter_indices_size = 0
 
         for i, batch in enumerate(batches):
             requests.extend(batch.requests)
@@ -592,6 +794,21 @@ class FlashCausalLMBatch(Batch):
             top_n_tokens_tensor[start_index:end_index] = batch.top_n_tokens_tensor
             slots[slots_start_index:slots_end_index] = batch.slots
 
+            # Copy over adapter indices
+            adapter_start_index = cumulative_adapter_indices_size
+            adapter_end_index = (
+                cumulative_adapter_indices_size
+                + batch.adapter_meta.adapter_indices.shape[0]
+            )
+            adapter_indices[adapter_start_index:adapter_end_index] = (
+                batch.adapter_meta.adapter_indices
+            )
+            cumulative_adapter_indices_size = adapter_end_index
+            adapter_set.update(batch.adapter_meta.adapter_set)
+            adapter_segment_builder.concat(
+                batch.adapter_meta.adapter_segments, batch.adapter_meta.segment_indices
+            )
+
             all_input_ids_tensor[
                 start_index:end_index, : batch.all_input_ids_tensor.shape[1]
             ] = batch.all_input_ids_tensor[:, :max_length]
@@ -600,10 +817,14 @@ class FlashCausalLMBatch(Batch):
                 start_index:end_index, : batch.block_tables_tensor.shape[1]
             ] = batch.block_tables_tensor[:, :max_blocks]
 
+            prefix_lens_tensor[start_index:end_index] = batch.prefix_lens_tensor
+
             start_slots.append(batch.start_slots + cumulative_slots)
 
             block_tables.extend(batch.block_tables)
+            prefix_lens.extend(batch.prefix_lens)
             all_input_ids.extend(batch.all_input_ids)
+            prefix_ids.extend(batch.prefix_ids)
 
             input_lengths.extend(batch.input_lengths)
             prefix_offsets.extend(batch.prefix_offsets)
@@ -621,6 +842,8 @@ class FlashCausalLMBatch(Batch):
 
         start_slots = torch.concat(start_slots)
 
+        # assert sum(len(b) for b in block_tables) == (block_tables_tensor != 0).sum()
+
         next_token_chooser = HeterogeneousNextTokenChooser.from_pb(
             next_token_chooser_parameters,
             dtype=batches[0].next_token_chooser.dtype,
@@ -635,10 +858,7 @@ class FlashCausalLMBatch(Batch):
             else None
         )
 
-        # Needed to avoid dropping blocks when the batches will go out of scope
-        for b in batches:
-            b.block_tables = None
-            del b
+        adapter_segments, adapter_segment_indices = adapter_segment_builder.build()
 
         return cls(
             batch_id=batches[0].batch_id,
@@ -647,11 +867,13 @@ class FlashCausalLMBatch(Batch):
             input_ids=input_ids,
             position_ids=position_ids,
             cu_seqlen_prefill=None,
+            prefill_cache_indices=None,
             start_slots=start_slots,
             slot_indices=slot_indices,
-            needed_blocks_slots=None,
             block_tables=block_tables,
             block_tables_tensor=block_tables_tensor,
+            prefix_lens=prefix_lens,
+            prefix_lens_tensor=prefix_lens_tensor,
             slots=slots,
             max_seqlen=max_seqlen,
             prefill_head_indices=None,
@@ -663,47 +885,176 @@ class FlashCausalLMBatch(Batch):
             read_offsets=read_offsets,
             all_input_ids=all_input_ids,
             all_input_ids_tensor=all_input_ids_tensor,
+            prefix_ids=prefix_ids,
             next_token_chooser=next_token_chooser,
             stopping_criterias=stopping_criterias,
             top_n_tokens=top_n_tokens,
             top_n_tokens_tensor=top_n_tokens_tensor,
-            blocks=blocks,
+            num_blocks=num_blocks,
             max_blocks=max_blocks,
             speculative_ids=speculative_ids,
+            adapter_meta=AdapterBatchMetadata(
+                adapter_indices=adapter_indices,
+                adapter_set=adapter_set,
+                adapter_segments=adapter_segments,
+                segment_indices=adapter_segment_indices,
+            ),
         )
 
-    def __del__(self):
-        if self.block_tables is not None and self.block_tables:
-            # Free blocks
-            get_cache_manager().free(
-                list(itertools.chain.from_iterable(self.block_tables))
-            )
-
     def __len__(self):
         return len(self.requests)
 
 
+ADAPTER_LAYERS = [
+    "q_proj",
+    "k_proj",
+    "v_proj",
+    "o_proj",
+    "gate_proj",
+    "up_proj",
+    "down_proj",
+]
+ROW_PARALLEL = {"o_proj", "down_proj", "lm_head"}
+
+
 class FlashCausalLM(Model):
     def __init__(
         self,
-        model: torch.nn.Module,
-        tokenizer: PreTrainedTokenizerBase,
-        num_layers: int,
-        num_kv_heads: int,
-        head_size: int,
-        dtype: torch.dtype,
-        device: torch.device,
-        rank: int = 0,
-        world_size: int = 1,
-        sliding_window: Optional[int] = None,
+        model_id: str,
+        model_class,
+        revision: Optional[str] = None,
+        quantize: Optional[str] = None,
+        speculator: Optional[str] = None,
+        dtype: Optional[torch.dtype] = None,
+        trust_remote_code: bool = False,
+        lora_adapter_ids: Optional[list] = [],
+        tokenizer_class: PreTrainedTokenizerBase = AutoTokenizer,
+        config_class: PreTrainedTokenizerBase = AutoConfig,
+        default_dtype=torch.float16,
+        aliases=None,
+        # Used for Santacoder override of config
+        num_kv_heads: Optional[int] = None,
+        # Deepseek V2 uses different QK and V dims.
+        head_size: Optional[int] = None,
+        skip_special_tokens: bool = True,
     ):
-        self.num_layers = num_layers
-        self.num_kv_heads = num_kv_heads
-        self.head_size = head_size
+        self.quantize = quantize
+        self.process_group, rank, world_size = initialize_torch_distributed()
+        if torch.cuda.is_available():
+            device = torch.device(f"cuda:{rank}")
+            dtype = default_dtype if dtype is None else dtype
+        elif SYSTEM == "ipex":
+            if hasattr(torch, "xpu") and torch.xpu.is_available():
+                device = torch.device(f"xpu:{rank}")
+                dtype = default_dtype if dtype is None else dtype
+            else:
+                device = torch.device("cpu")
+                # Float16 doesn't exist on target.
+                dtype = torch.bfloat16 if dtype is None else dtype
+                init_cpu_threads_env(rank_id=rank, world_size=world_size)
+        else:
+            raise NotImplementedError(f"{model_class} is only available on GPU")
+
+        tokenizer = tokenizer_class.from_pretrained(
+            model_id,
+            revision=revision,
+            padding_side="left",
+            truncation_side="left",
+            trust_remote_code=trust_remote_code,
+        )
+        try:
+            generation_config = GenerationConfig.from_pretrained(
+                model_id, revision=revision, trust_remote_code=trust_remote_code
+            )
+            if isinstance(generation_config.eos_token_id, (list, set)):
+                # TODO Huge hack
+                tokenizer._eos_token_ids = set(generation_config.eos_token_id)
+        except Exception:
+            pass
+
+        config = config_class.from_pretrained(
+            model_id, revision=revision, trust_remote_code=trust_remote_code
+        )
+        config.quantize = quantize
+        config.speculator = speculator
+
+        torch.distributed.barrier(group=self.process_group)
+
+        weights_loader = get_loader(quantize, model_id, revision)
+        filenames = weight_files(model_id, revision=revision, extension=".safetensors")
+        weights = Weights(
+            filenames,
+            device,
+            dtype,
+            process_group=self.process_group,
+            aliases=aliases,
+            weights_loader=weights_loader,
+        )
+
+        prefix = ""
+        model = model_class(prefix, config, weights)
+        torch.distributed.barrier(group=self.process_group)
+
+        # VLM models define the config we care about in their text_config
+        text_config = getattr(config, "text_config", None)
+        if text_config is not None:
+            config = text_config
+
+        if getattr(config, "sliding_window", None) is not None:
+            set_sliding_window(config.sliding_window)
+        else:
+            config.sliding_window = None
+
+        self.num_layers = config.num_hidden_layers
+        self.num_heads = config.num_attention_heads // self.process_group.size()
+        # Validation is done in the model itself
+        if num_kv_heads is None:
+            num_kv_heads = getattr(config, "num_key_value_heads", None)
+            # GPT-2 workaround
+            if num_kv_heads is None:
+                num_kv_heads = getattr(config, "n_head", None)
+        if num_kv_heads is None:
+            raise ValueError("Cannot get the number of key/value heads")
+        self.num_kv_heads = (
+            num_kv_heads // self.process_group.size()
+            if num_kv_heads > 1
+            else num_kv_heads
+        )
+        assert self.num_kv_heads > 0
+
+        if head_size is None:
+            # Some models use GQA and different sizes for o_proj
+            # and q_proj, that allows for that.
+            if hasattr(config, "head_dim"):
+                self.head_size = config.head_dim
+            else:
+                self.head_size = config.hidden_size // config.num_attention_heads
+        else:
+            self.head_size = head_size
 
         self.cuda_graphs = {}
+        self.kv_cache = []
 
-        super(FlashCausalLM, self).__init__(
+        if ATTENTION == "flashinfer":
+            from text_generation_server.layers.attention.flashinfer import (
+                create_prefill_state,
+                create_decode_state,
+                create_prefill_with_paged_kv_state,
+            )
+
+            self.prefill_state = create_prefill_state(device=device)
+            self.prefill_with_paged_kv_state = create_prefill_with_paged_kv_state(
+                device=device
+            )
+
+            self.decode_state = create_decode_state(
+                device=device,
+                num_heads=self.num_heads,
+                num_kv_heads=self.num_kv_heads,
+            )
+
+        super().__init__(
+            model_id=model_id,
             model=model,
             tokenizer=tokenizer,
             requires_padding=False,
@@ -711,65 +1062,190 @@ class FlashCausalLM(Model):
             device=device,
             rank=rank,
             world_size=world_size,
-            sliding_window=sliding_window,
+            sliding_window=config.sliding_window,
         )
 
     @property
     def batch_type(self) -> Type[FlashCausalLMBatch]:
         return FlashCausalLMBatch
 
+    def max_past(self) -> int:
+        return getattr(self.model, "max_past", None)
+
+    def init_kv_cache(
+        self,
+        num_blocks: int,
+        num_layers: int,
+        num_heads: int,
+        head_size: int,
+        dtype: torch.dtype,
+        device: torch.device,
+    ):
+        self.kv_cache = []
+        empty_cache()
+
+        element_size = torch.tensor([], dtype=dtype).element_size()
+        if SYSTEM == "ipex" and device.type == "xpu":
+            x = 1
+        else:
+            x = BLOCK_SIZE // element_size
+
+        if ATTENTION in {"flashdecoding", "flashinfer"}:
+            self.kv_cache = [
+                (
+                    torch.empty(
+                        (num_blocks, BLOCK_SIZE, num_heads, head_size),
+                        dtype=dtype,
+                        device=device,
+                    ),
+                    torch.empty(
+                        (num_blocks, BLOCK_SIZE, num_heads, head_size),
+                        dtype=dtype,
+                        device=device,
+                    ),
+                )
+                for _ in range(num_layers)
+            ]
+        elif SYSTEM == "ipex" and device == torch.device("cpu"):
+            self.kv_cache = [
+                (
+                    torch.empty(
+                        (num_blocks, num_heads, BLOCK_SIZE, head_size),
+                        dtype=dtype,
+                        device=device,
+                    ),
+                    torch.empty(
+                        (num_blocks, num_heads, BLOCK_SIZE, head_size),
+                        dtype=dtype,
+                        device=device,
+                    ),
+                )
+                for _ in range(num_layers)
+            ]
+        else:
+            self.kv_cache = [
+                (
+                    torch.zeros(
+                        (num_blocks, num_heads, head_size // x, BLOCK_SIZE, x),
+                        dtype=dtype,
+                        device=device,
+                    ),
+                    torch.zeros(
+                        (num_blocks, num_heads, head_size, BLOCK_SIZE),
+                        dtype=dtype,
+                        device=device,
+                    ),
+                )
+                for _ in range(num_layers)
+            ]
+
     def cuda_graph_warmup(self, bs: int, max_s: int, max_bt: int):
         input_ids = torch.zeros(bs, dtype=torch.int64, device=self.device)
         position_ids = torch.zeros(bs, dtype=torch.int32, device=self.device)
         slots = torch.arange(bs, dtype=torch.int64, device=self.device)
-        input_lengths = torch.ones(bs, dtype=torch.int32, device=self.device) * max_s
-        block_tables = (
-            torch.arange(max_bt, dtype=torch.int32, device=self.device)
-            .repeat(bs)
-            .reshape((bs, max_bt))
+        input_lengths = [max_s] * bs
+        prefix_lengths = [0] * bs
+        input_lengths_tensor = (
+            torch.ones(bs, dtype=torch.int32, device=self.device) * max_s
         )
-        kv_cache = get_cache_manager().kv_cache
+        prefix_lengths_tensor = torch.zeros(bs, dtype=torch.int32, device=self.device)
+        block_tables = torch.arange(
+            max_bt, dtype=torch.int32, device=self.device
+        ).repeat(bs)
+        block_tables = block_tables.reshape((bs, max_bt))
 
+        if ATTENTION == "flashinfer":
+            block_tables = block_tables_to_ragged(
+                block_tables=block_tables,
+                input_lengths=input_lengths,
+                prefix_lens=prefix_lengths,
+            )
+            from text_generation_server.layers.attention.flashinfer import (
+                create_decode_state_cuda_graphs,
+            )
+
+            block_tables_ptr = torch.zeros(
+                bs + 1, dtype=torch.int32, device=self.device
+            )
+            last_page_len = torch.ones(bs, dtype=torch.int32, device=self.device)
+            state = create_decode_state_cuda_graphs(
+                device=input_ids.device,
+                block_tables=block_tables,
+                block_tables_ptr=block_tables_ptr,
+                last_page_len=last_page_len,
+                num_heads=self.num_heads,
+                num_kv_heads=self.num_kv_heads,
+            )
+        else:
+            state = None
+
+        graph = torch.cuda.CUDAGraph()
         self.cuda_graphs[bs] = {
             "input_ids": input_ids,
             "position_ids": position_ids,
-            "kv_cache": kv_cache,
+            "kv_cache": self.kv_cache,
             "block_tables": block_tables,
             "slots": slots,
-            "input_lengths": input_lengths,
+            "input_lengths": input_lengths_tensor,
+            "prefix_lengths": prefix_lengths_tensor,
+            "state": state,
+            "graph": graph,
         }
-        graph = torch.cuda.CUDAGraph()
-        self.cuda_graphs[bs]["graph"] = graph
 
         torch.cuda.synchronize()
         # Run once outside to warmup
-        self.model.forward(
-            input_ids=input_ids,
-            position_ids=position_ids,
-            cu_seqlen_prefill=None,
-            kv_cache=kv_cache,
+        with self._forward_context(
             block_tables=block_tables,
-            slots=slots,
-            input_lengths=input_lengths,
-            max_s=max_s,
-            lm_head_indices=None,
-        )
-        torch.cuda.synchronize()
-
-        with torch.cuda.graph(graph, pool=MEM_POOL):
-            logits, speculative_logits = self.model.forward(
+            cu_seqlen_prefill=None,
+            input_lengths_tensor=input_lengths_tensor,
+            state=state,
+            prefix_lens_tensor=prefix_lengths_tensor,
+        ):
+            seqlen = Seqlen(
+                input_lengths=input_lengths_tensor,
+                prefix_lengths=prefix_lengths_tensor,
+                cu_seqlen_q=None,
+                max_q=1,
+                max_k=max_s,
+            )
+            self.model.forward(
                 input_ids=input_ids,
                 position_ids=position_ids,
                 cu_seqlen_prefill=None,
-                kv_cache=kv_cache,
+                kv_cache=self.kv_cache,
                 block_tables=block_tables,
                 slots=slots,
-                input_lengths=input_lengths,
+                seqlen=seqlen,
                 max_s=max_s,
+                prefill_cache_indices=None,
                 lm_head_indices=None,
             )
-            self.cuda_graphs[bs]["logits"] = logits
-            self.cuda_graphs[bs]["speculative_logits"] = speculative_logits
+            del seqlen
+
+            torch.cuda.synchronize()
+
+            with torch.cuda.graph(graph, pool=MEM_POOL):
+                seqlen = Seqlen(
+                    input_lengths=input_lengths_tensor,
+                    prefix_lengths=prefix_lengths_tensor,
+                    cu_seqlen_q=None,
+                    max_q=1,
+                    max_k=max_s,
+                )
+                logits, speculative_logits = self.model.forward(
+                    input_ids=input_ids,
+                    position_ids=position_ids,
+                    cu_seqlen_prefill=None,
+                    kv_cache=self.kv_cache,
+                    block_tables=block_tables,
+                    slots=slots,
+                    seqlen=seqlen,
+                    max_s=max_s,
+                    prefill_cache_indices=None,
+                    lm_head_indices=None,
+                )
+                self.cuda_graphs[bs]["logits"] = logits
+                self.cuda_graphs[bs]["speculative_logits"] = speculative_logits
         torch.cuda.synchronize()
 
     def warmup(self, batch: FlashCausalLMBatch):
@@ -777,17 +1253,16 @@ class FlashCausalLM(Model):
         empty_cache()
 
         try:
-            cache_manager = set_cache_manager(
-                batch.blocks,
+            self.init_kv_cache(
+                batch.num_blocks,
                 self.num_layers,
                 self.num_kv_heads,
                 self.head_size,
-                self.sliding_window is not None,
                 self.dtype,
                 self.device,
             )
             max_bt = batch.max_blocks
-            max_s = max_bt * get_cache_manager().block_size
+            max_s = max_bt * BLOCK_SIZE
 
             if SYSTEM == "rocm" and os.environ.get("PYTORCH_TUNABLEOP_ENABLED", False):
                 torch.cuda.tunable.tuning_enable(False)
@@ -807,23 +1282,22 @@ class FlashCausalLM(Model):
         total_cache_size = self.num_layers * cache_block_size * 2 * dtype_size
 
         free_memory = get_free_memory(self.device, MEMORY_FRACTION)
+        batch_num_blocks = batch.num_blocks if batch is not None else 0
 
         num_blocks = (
             # Leave 5% for some wiggle room
-            int((free_memory * 0.95) // total_cache_size)
-            # Add batch.blocks as we allocated it above, so it is included in the peak memory.
-            + cache_manager.num_blocks
+            int((free_memory * TGI_WIGGLE_ROOM) // total_cache_size)
+            # Add batch.num_blocks as we allocated it above, so it is included in the peak memory.
+            + batch_num_blocks
         )
 
         del batch
-        del cache_manager
 
-        set_cache_manager(
+        self.init_kv_cache(
             num_blocks,
             self.num_layers,
             self.num_kv_heads,
             self.head_size,
-            self.sliding_window is not None,
             self.dtype,
             self.device,
         )
@@ -833,6 +1307,8 @@ class FlashCausalLM(Model):
                 os.environ.get("PYTORCH_TUNABLEOP_ENABLED") is None
                 or os.environ.get("PYTORCH_TUNABLEOP_ENABLED") == "1"
             ):
+                torch.cuda.tunable.enable()
+
                 if os.environ.get("PYTORCH_TUNABLEOP_TUNING") != "0":
                     torch.cuda.tunable.tuning_enable(True)
 
@@ -841,47 +1317,61 @@ class FlashCausalLM(Model):
                         int(val)
                         for val in os.environ["PYTORCH_TUNABLEOP_SEQLENS"].split(",")
                     ]
-                else:
+                elif CUDA_GRAPHS is not None:
                     tuning_sequences = CUDA_GRAPHS
+                else:
+                    tuning_sequences = [1, 2, 3, 4, 5, 6, 7]
 
                 tunableop_filepath = os.path.join(
                     HUGGINGFACE_HUB_CACHE,
-                    f"tunableop_{tgi_globals.MODEL_ID.replace('/', '-')}_tp{self.world_size}_rank{self.rank}.csv",
+                    f"tunableop_{self.model_id.replace('/', '-')}_tp{self.world_size}_rank{self.rank}.csv",
                 )
 
-                logger.info(
-                    f"PyTorch TunableOp (https://github.com/fxmarty/pytorch/tree/2.3-patched/aten/src/ATen/cuda/tunable) is enabled. The warmup may take several minutes, picking the ROCm optimal matrix multiplication kernel for the target lengths {', '.join([str(seqlen) for seqlen in tuning_sequences])}, with typical 5-8% latency improvement for small sequence lengths. The picked GEMMs are saved in the file {tunableop_filepath}. To disable TunableOp, please launch TGI with `PYTORCH_TUNABLEOP_ENABLED=0`."
+                log_master(
+                    logger.info,
+                    f"PyTorch TunableOp is enabled. The warmup may take several minutes, picking the ROCm optimal matrix multiplication kernel for the target lengths {', '.join([str(seqlen) for seqlen in tuning_sequences])}, with typical 5-8% latency improvement for small sequence lengths. The picked GEMMs are saved in the file {tunableop_filepath}. To disable TunableOp, please launch TGI with `PYTORCH_TUNABLEOP_ENABLED=0`.",
+                )
+
+                torch.cuda.tunable.set_filename(
+                    tunableop_filepath, insert_device_ordinal=False
                 )
 
                 if os.path.isfile(tunableop_filepath):
-                    logger.info(
-                        f"The file {tunableop_filepath} already exists and will be reused."
+                    log_master(
+                        logger.info,
+                        f"The file {tunableop_filepath} already exists and will be reused.",
                     )
                     torch.cuda.tunable.read_file(tunableop_filepath)
 
                 os.makedirs(HUGGINGFACE_HUB_CACHE, exist_ok=True)
 
                 for seqlen in tuning_sequences:
-                    logger.info(f"Warming up TunableOp for seqlen={seqlen}")
+                    log_master(logger.info, f"Warming up TunableOp for seqlen={seqlen}")
                     self.tunableop_warmup(seqlen)
                     torch.cuda.tunable.write_file(tunableop_filepath)
-                torch.cuda.tunable.tuning_enable(False)
+                if os.environ.get("PYTORCH_TUNABLEOP_TUNING_AFTER_WARMUP") != "1":
+                    torch.cuda.tunable.tuning_enable(False)
             else:
-                logger.info(
-                    "PyTorch ROCm TunableOp (https://github.com/pytorch/pytorch/tree/main/aten/src/ATen/cuda/tunable) is disabled. TunableOp brings an additional 5-8% latency improvement for small sequence lengths but requires a warmup. If necessary, please use the environment variable PYTORCH_TUNABLEOP_ENABLED=1 to enable TunableOp."
+                log_master(
+                    logger.info,
+                    "PyTorch ROCm TunableOp (https://github.com/pytorch/pytorch/tree/main/aten/src/ATen/cuda/tunable) is disabled. TunableOp brings an additional 5-8% latency improvement for small sequence lengths but requires a warmup. If necessary, please use the environment variable PYTORCH_TUNABLEOP_ENABLED=1 to enable TunableOp.",
                 )
 
         if CUDA_GRAPHS:
             try:
-                logger.info(f"Cuda Graphs are enabled for sizes {CUDA_GRAPHS}")
+                log_master(
+                    logger.info, f"Cuda Graphs are enabled for sizes {CUDA_GRAPHS}"
+                )
                 # Warmup cuda graphs
                 for bs in CUDA_GRAPHS:
                     if self.speculate is None or self.speculate + 1 <= bs:
                         self.cuda_graph_warmup(bs, max_s, max_bt)
             except torch.cuda.OutOfMemoryError:
-                logger.exception(f"Decode cuda graph warmup failed")
+                logger.exception("Decode cuda graph warmup failed")
         else:
-            logger.info(f"Cuda Graphs are disabled (CUDA_GRAPHS={CUDA_GRAPHS}).")
+            log_master(
+                logger.info, f"Cuda Graphs are disabled (CUDA_GRAPHS={CUDA_GRAPHS})."
+            )
 
         return int(num_blocks * BLOCK_SIZE)
 
@@ -889,35 +1379,45 @@ class FlashCausalLM(Model):
         input_ids = torch.zeros(seqlen, dtype=torch.int64, device=self.device)
         position_ids = torch.zeros(seqlen, dtype=torch.int32, device=self.device)
         slots = torch.arange(seqlen, dtype=torch.int64, device=self.device)
-        kv_cache = get_cache_manager().kv_cache
 
         # Dummy value, some models (starcoder2) don't accept `None`.
         input_lengths = torch.ones(seqlen, dtype=torch.int32, device=self.device)
+        prefix_lens_tensor = torch.zeros(seqlen, dtype=torch.int32, device=self.device)
+        cu_seqlen_prefill = torch.tensor(
+            [0, seqlen], device=self.device, dtype=torch.int32
+        )
+        max_s = seqlen
+        seqlen = Seqlen(
+            input_lengths=input_lengths,
+            prefix_lengths=prefix_lens_tensor,
+            cu_seqlen_q=cu_seqlen_prefill,
+            max_q=1,
+            max_k=seqlen,
+        )
 
         # We pass a `cu_seqlen_prefill` in order not to have to deal with paged attention cache allocation/deallocation.
         self.model.forward(
             input_ids=input_ids,
             position_ids=position_ids,
-            cu_seqlen_prefill=torch.tensor(
-                [0, seqlen], device=self.device, dtype=torch.int32
-            ),
-            kv_cache=get_cache_manager().kv_cache,
+            cu_seqlen_prefill=cu_seqlen_prefill,
+            kv_cache=self.kv_cache,
             block_tables=None,
-            input_lengths=input_lengths,
+            seqlen=seqlen,
             slots=slots,
-            max_s=seqlen,
+            max_s=max_s,
             lm_head_indices=None,
+            prefill_cache_indices=None,
         )
 
     def forward(
-        self, batch: FlashCausalLMBatch
+        self, batch: FlashCausalLMBatch, adapter_data: AdapterBatchData
     ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
         # Model Forward
         if batch.speculative_ids is not None:
             input_ids = batch.input_ids
             position_ids = batch.position_ids
             cu_seqlen_prefill = batch.cu_seqlen_prefill
-            kv_cache = get_cache_manager().kv_cache
+            kv_cache = self.kv_cache
             block_tables = batch.block_tables_tensor
             slots = batch.slots[batch.slot_indices]
             input_lengths = batch.input_lengths_tensor
@@ -940,6 +1440,9 @@ class FlashCausalLM(Model):
             input_lengths = (
                 input_lengths.unsqueeze(-1).expand(B, new_length) + arange_int
             ).view(-1)
+            prefix_lens_tensor = (
+                batch.prefix_lens_tensor.unsqueeze(-1).expand(B, new_length)
+            ).reshape(-1)
 
             # Add Copy the block tables for all members
             block_tables = (
@@ -956,13 +1459,20 @@ class FlashCausalLM(Model):
             input_ids = batch.input_ids
             position_ids = batch.position_ids
             cu_seqlen_prefill = batch.cu_seqlen_prefill
-            kv_cache = get_cache_manager().kv_cache
+            kv_cache = self.kv_cache
             block_tables = batch.block_tables_tensor
             slots = batch.slots[batch.slot_indices]
             input_lengths = batch.input_lengths_tensor
+            prefix_lens_tensor = batch.prefix_lens_tensor
             max_s = batch.max_seqlen
             lm_head_indices = batch.prefill_head_indices
 
+        if cu_seqlen_prefill is None and self.max_past() is not None:
+            # In decode, not prefill, we're actually overwriting the KV-cache
+            # in a circular buffer mode.
+            # This makes sure the max_s for the decode pass is correct.
+            max_s = min(self.max_past(), max_s)
+
         bs = input_ids.shape[0]
         sorted_padded_bs = sorted([k for k in self.cuda_graphs.keys() if k >= bs])
         if sorted_padded_bs:
@@ -972,32 +1482,79 @@ class FlashCausalLM(Model):
             cuda_graph = None
 
         if cu_seqlen_prefill is not None or cuda_graph is None:
-            return self.model.forward(
-                input_ids=input_ids,
-                position_ids=position_ids,
-                cu_seqlen_prefill=cu_seqlen_prefill,
-                kv_cache=kv_cache,
+            if ATTENTION == "flashinfer":
+                block_tables = block_tables_to_ragged(
+                    block_tables=block_tables,
+                    input_lengths=batch.input_lengths,
+                    prefix_lens=batch.prefix_lens,
+                )
+            with self._forward_context(
                 block_tables=block_tables,
-                slots=slots,
-                input_lengths=input_lengths,
-                max_s=max_s,
-                lm_head_indices=lm_head_indices,
-            )
+                cu_seqlen_prefill=cu_seqlen_prefill,
+                input_lengths_tensor=input_lengths,
+                prefix_lens_tensor=prefix_lens_tensor,
+            ):
+                max_k = (input_lengths + prefix_lens_tensor).max().item()
+                seqlen = Seqlen(
+                    input_lengths=input_lengths,
+                    prefix_lengths=prefix_lens_tensor,
+                    cu_seqlen_q=cu_seqlen_prefill,
+                    max_q=max_s,
+                    max_k=max_k,
+                )
+                logits, speculative_logits = self.model.forward(
+                    input_ids=input_ids,
+                    position_ids=position_ids,
+                    cu_seqlen_prefill=cu_seqlen_prefill,
+                    kv_cache=kv_cache,
+                    block_tables=block_tables,
+                    slots=slots,
+                    seqlen=seqlen,
+                    max_s=max_s,
+                    prefill_cache_indices=batch.prefill_cache_indices,
+                    lm_head_indices=lm_head_indices,
+                    adapter_data=adapter_data,
+                )
+                if batch.prefill_cache_indices is not None:
+                    batch.prefill_cache_indices = None
+                return logits, speculative_logits
 
         # Copy inputs to the static inputs of the cuda graph
         # Static inputs are potentially padded
         cuda_graph["input_ids"][: input_ids.shape[0]] = input_ids
         cuda_graph["position_ids"][: position_ids.shape[0]] = position_ids
-        cuda_graph["block_tables"][
-            : block_tables.shape[0], : block_tables.shape[1]
-        ] = block_tables
-        cuda_graph["slots"].fill_(-1)
+        if ATTENTION == "flashinfer":
+            block_tables = block_tables_to_ragged(
+                block_tables=block_tables,
+                input_lengths=batch.input_lengths,
+                prefix_lens=batch.prefix_lens,
+            )
+            # assert block_tables.shape[0] >= slots.shape[0]
+            cuda_graph["block_tables"][: block_tables.shape[0]] = block_tables
+        else:
+            cuda_graph["block_tables"][
+                : block_tables.shape[0], : block_tables.shape[1]
+            ] = block_tables
+
+        # XXX: This is working only because block 0 is reserved for the healthcheck
+        # so it doesn't matter if we override it with bogus values.
+        cuda_graph["slots"].fill_(0)
         cuda_graph["slots"][: slots.shape[0]] = slots
         cuda_graph["input_lengths"].zero_()
         cuda_graph["input_lengths"][: input_lengths.shape[0]] = input_lengths
+        cuda_graph["prefix_lengths"].zero_()
+        cuda_graph["prefix_lengths"][: prefix_lens_tensor.shape[0]] = prefix_lens_tensor
+
+        with self._forward_context(
+            block_tables=cuda_graph["block_tables"],
+            cu_seqlen_prefill=None,
+            input_lengths_tensor=cuda_graph["input_lengths"],
+            prefix_lens_tensor=cuda_graph["prefix_lengths"],
+            state=cuda_graph["state"],
+        ):
+            # Replay the graph
+            cuda_graph["graph"].replay()
 
-        # Replay the graph
-        cuda_graph["graph"].replay()
         # Slice output to the correct shape
         speculative_logits = (
             cuda_graph["speculative_logits"][:bs]
@@ -1015,24 +1572,34 @@ class FlashCausalLM(Model):
         prefill = batch.cu_seqlen_prefill is not None
         prefill_logprobs = batch.prefill_next_token_indices is not None
 
-        if batch.needed_blocks_slots:
-            # Allocate blocks to this batch
-            block_tables, block_tables_tensor, slots = get_cache_manager().allocate(
-                batch.needed_blocks_slots,
-                batch.blocks,
-                batch.max_blocks,
-                batch.input_ids.device,
+        # Update adapter indices for speculative tokens (if present)
+        adapter_meta = batch.adapter_meta
+        if batch.speculative_ids is not None:
+            B, speculative_length = batch.speculative_ids.shape
+            new_length = speculative_length + 1
+            adapter_indices = (
+                adapter_meta.adapter_indices.unsqueeze(-1)
+                .expand(B, new_length)
+                .reshape(-1)
+            )
+            adapter_segments = adapter_meta.adapter_segments * new_length
+            adapter_meta = AdapterBatchMetadata(
+                adapter_indices=adapter_indices,
+                adapter_set=adapter_meta.adapter_set,
+                adapter_segments=adapter_segments,
+                segment_indices=adapter_meta.segment_indices,
             )
-            batch.needed_blocks_slots = None
-            batch.block_tables = block_tables
-            batch.block_tables_tensor = block_tables_tensor
-            batch.slots = slots
 
-        try:
-            out, speculative_logits = self.forward(batch)
-        except Exception as e:
-            del batch
-            raise e
+        # Assign pointers to adapter weights
+        # TODO(travis): don't update this if indices haven't changed
+        adapter_data = AdapterBatchData.from_meta(
+            adapter_meta,
+            self.layer_to_adapter_weights,
+            prefill,
+            batch.prefill_head_indices,
+        )
+
+        out, speculative_logits = self.forward(batch, adapter_data)
 
         if prefill:
             next_token_logits = (
@@ -1044,8 +1611,13 @@ class FlashCausalLM(Model):
                     if prefill_logprobs
                     else speculative_logits
                 )
+            next_adapter_indices = batch.adapter_meta.adapter_indices.new_empty(
+                len(batch)
+            )
+
         else:
             next_token_logits = out
+            next_adapter_indices = batch.adapter_meta.adapter_indices
 
         speculate = get_speculate()
         (
@@ -1111,6 +1683,12 @@ class FlashCausalLM(Model):
                 # In decode, we do not need this as we can just increment position ids
                 next_position_ids[i] = batch.position_ids[end_index - 1]
 
+                # Initialize adapter indices
+                # In decode, we only have one token per row in the batch, so grab last index
+                next_adapter_indices[i] = batch.adapter_meta.adapter_indices[
+                    end_index - 1
+                ]
+
                 # Used to gather prefill logprobs
                 # Copy batch.input_ids to prefill_token_indices
                 if prefill_logprobs:
@@ -1136,6 +1714,16 @@ class FlashCausalLM(Model):
         batch.position_ids = next_position_ids + accepted_ids
         batch.input_lengths_tensor += accepted_ids
         batch.slot_indices += accepted_ids
+        batch.adapter_meta.adapter_indices = next_adapter_indices
+
+        if prefill:
+            # adjust segment lengths to account for all request lengths being 1 during decoding
+            adapter_segments, _ = find_segments(batch.adapter_meta.adapter_indices)
+            batch.adapter_meta.adapter_segments = torch.tensor(
+                adapter_segments,
+                dtype=torch.int32,
+                device=batch.adapter_meta.adapter_segments.device,
+            )
 
         if prefill and prefill_logprobs:
             # Get prefill logprobs
@@ -1160,6 +1748,7 @@ class FlashCausalLM(Model):
             batch.read_offsets,
             batch.stopping_criterias,
             batch.all_input_ids,
+            batch.prefix_ids,
             batch.next_token_chooser.do_sample,
             batch.next_token_chooser.seeds,
             batch.top_n_tokens,
@@ -1177,6 +1766,7 @@ class FlashCausalLM(Model):
             read_offset,
             stopping_criteria,
             all_input_ids,
+            prefix_ids,
             do_sample,
             seed,
             top_n_tokens,
@@ -1189,8 +1779,7 @@ class FlashCausalLM(Model):
             left = 0
 
             if n_accepted_ids > 1:
-                if RANK == 0:
-                    logger.debug(f"Speculated ids {n_accepted_ids - 1}")
+                log_master(logger.debug, f"speculated ids {n_accepted_ids - 1}")
 
             current_stopped = False
             for j in range(index, index + n_accepted_ids):
@@ -1252,18 +1841,18 @@ class FlashCausalLM(Model):
                     out_end_index = batch.prefill_cu_outlens[i + 1]
 
                     # Remove generated token to only have prefill and add nan for first prompt token
-                    request_prefill_logprobs = [float("nan")] + prefill_logprobs[
-                        out_start_index : out_end_index - 1
-                    ]
+                    request_prefill_logprobs = (
+                        [float("nan")] * (len(prefix_ids) + 1)
+                    ) + prefill_logprobs[out_start_index : out_end_index - 1]
                     prefill_token_ids = all_input_ids[:-1]
                     prefill_texts = self.tokenizer.batch_decode(
-                        prefill_token_ids,
+                        prefix_ids + prefill_token_ids,
                         clean_up_tokenization_spaces=False,
                         skip_special_tokens=False,
                     )
 
                     prefill_tokens = Tokens(
-                        prefill_token_ids,
+                        prefix_ids + prefill_token_ids,
                         request_prefill_logprobs,
                         prefill_texts,
                         is_special=[],
@@ -1327,7 +1916,6 @@ class FlashCausalLM(Model):
             batch.all_input_ids[i] = all_input_ids
 
         if stopped:
-            del batch
             # No need to return a batch if we know that all requests stopped
             forward_ns = start_decode - start
             decode_ns = time.time_ns() - start_decode
@@ -1340,3 +1928,76 @@ class FlashCausalLM(Model):
         forward_ns = start_decode - start
         decode_ns = time.time_ns() - start_decode
         return generations, batch, (forward_ns, decode_ns)
+
+    def _forward_context(
+        self,
+        *,
+        block_tables: torch.Tensor,
+        cu_seqlen_prefill: Optional[torch.Tensor],
+        input_lengths_tensor: torch.Tensor,
+        prefix_lens_tensor: torch.Tensor,
+        state: Optional[Any] = None,
+    ) -> ContextManager:
+        if ATTENTION != "flashinfer":
+            return nullcontext()
+
+        from text_generation_server.layers.attention.flashinfer import (
+            use_decode_state,
+            use_prefill_with_paged_kv_state,
+        )
+
+        # has_prefix_lens = any(prefix_len > 0 for prefix_len in prefix_lens)
+
+        if cu_seqlen_prefill is not None:
+            return use_prefill_with_paged_kv_state(
+                state=(
+                    state if state is not None else self.prefill_with_paged_kv_state
+                ),
+                # block_tables=block_tables_to_ragged(
+                #     block_tables=block_tables,
+                #     input_lengths=input_lengths,
+                #     prefix_lens=prefix_lens,
+                # ),
+                block_tables=block_tables,
+                cu_seqlens=cu_seqlen_prefill,
+                input_lengths=input_lengths_tensor + prefix_lens_tensor,
+                num_heads=self.num_heads,
+                num_kv_heads=self.num_kv_heads,
+                head_size=self.head_size,
+                page_size=BLOCK_SIZE,
+                dtype=self.dtype,
+                window_left=self.sliding_window,
+            )
+        else:
+            assert input_lengths_tensor is not None
+            return use_decode_state(
+                state=state if state is not None else self.decode_state,
+                input_lengths=input_lengths_tensor + prefix_lens_tensor,
+                block_tables=block_tables,
+                num_heads=self.num_heads,
+                num_kv_heads=self.num_kv_heads,
+                head_size=self.head_size,
+                page_size=BLOCK_SIZE,
+                dtype=self.dtype,
+                window_left=self.sliding_window,
+            )
+
+
+def block_tables_to_ragged(
+    *, block_tables: torch.Tensor, input_lengths: List[int], prefix_lens: List[int]
+) -> torch.Tensor:
+    """Convert block table to ragged format compatible with FlashInfer."""
+    assert len(input_lengths) == len(prefix_lens)
+
+    total_len = sum(input_lengths) + sum(prefix_lens)
+    block_tables_ragged = torch.empty(
+        total_len, dtype=torch.int32, device=block_tables.device
+    )
+
+    offset = 0
+    for i, (input_length, prefix_len) in enumerate(zip(input_lengths, prefix_lens)):
+        seq_len = prefix_len + input_length
+        block_tables_ragged[offset : offset + seq_len] = block_tables[i][:seq_len]
+        offset += seq_len
+
+    return block_tables_ragged
diff --git a/server/text_generation_server/models/flash_cohere.py b/server/text_generation_server/models/flash_cohere.py
deleted file mode 100644
index b907ee08..00000000
--- a/server/text_generation_server/models/flash_cohere.py
+++ /dev/null
@@ -1,74 +0,0 @@
-import torch
-import torch.distributed
-
-from opentelemetry import trace
-from typing import Optional
-from transformers import AutoTokenizer, AutoConfig
-
-from text_generation_server.models import FlashCausalLM
-from text_generation_server.models.custom_modeling.flash_cohere_modeling import (
-    FlashCohereForCausalLM,
-)
-from text_generation_server.utils import (
-    initialize_torch_distributed,
-    weight_files,
-    Weights,
-)
-
-tracer = trace.get_tracer(__name__)
-
-
-class FlashCohere(FlashCausalLM):
-    def __init__(
-        self,
-        model_id: str,
-        revision: Optional[str] = None,
-        quantize: Optional[str] = None,
-        speculator: Optional[str] = None,
-        dtype: Optional[torch.dtype] = None,
-        trust_remote_code: bool = False,
-    ):
-        self.process_group, rank, world_size = initialize_torch_distributed()
-        if torch.cuda.is_available():
-            device = torch.device(f"cuda:{rank}")
-            dtype = torch.float16 if dtype is None else dtype
-        else:
-            raise NotImplementedError("FlashCohere is only available on GPU")
-
-        tokenizer = AutoTokenizer.from_pretrained(
-            model_id,
-            revision=revision,
-            padding_side="left",
-            truncation_side="left",
-            trust_remote_code=trust_remote_code,
-            use_fast=True,
-            from_slow=False,
-        )
-
-        config = AutoConfig.from_pretrained(
-            model_id, revision=revision, trust_remote_code=trust_remote_code
-        )
-        config.quantize = quantize
-        config.speculator = speculator
-
-        torch.distributed.barrier(group=self.process_group)
-
-        filenames = weight_files(model_id, revision=revision, extension=".safetensors")
-        weights = Weights(filenames, device, dtype, process_group=self.process_group)
-        if config.quantize in ["gptq", "awq"]:
-            weights._set_gptq_params(model_id, revision)
-
-        model = FlashCohereForCausalLM(config, weights)
-
-        torch.distributed.barrier(group=self.process_group)
-        super(FlashCohere, self).__init__(
-            model=model,
-            tokenizer=tokenizer,
-            num_layers=len(model.model.layers),
-            num_kv_heads=model.model.num_key_value_heads,
-            head_size=model.model.head_size,
-            dtype=dtype,
-            device=device,
-            rank=rank,
-            world_size=world_size,
-        )
diff --git a/server/text_generation_server/models/flash_dbrx.py b/server/text_generation_server/models/flash_dbrx.py
deleted file mode 100644
index d5eb1a6e..00000000
--- a/server/text_generation_server/models/flash_dbrx.py
+++ /dev/null
@@ -1,99 +0,0 @@
-import torch
-import torch.distributed
-
-from opentelemetry import trace
-from typing import Optional
-from transformers import AutoTokenizer
-from transformers.models.gpt2 import GPT2TokenizerFast
-
-from text_generation_server.models import FlashCausalLM
-from text_generation_server.models.custom_modeling.flash_dbrx_modeling import (
-    FlashDbrxForCausalLM,
-    DbrxConfig,
-)
-from text_generation_server.utils import (
-    initialize_torch_distributed,
-    weight_files,
-    Weights,
-)
-
-tracer = trace.get_tracer(__name__)
-
-
-class FlashDbrx(FlashCausalLM):
-    def __init__(
-        self,
-        model_id: str,
-        revision: Optional[str] = None,
-        quantize: Optional[str] = None,
-        speculator: Optional[str] = None,
-        dtype: Optional[torch.dtype] = None,
-        trust_remote_code: bool = False,
-    ):
-        self.process_group, rank, world_size = initialize_torch_distributed()
-        if torch.cuda.is_available():
-            device = torch.device(f"cuda:{rank}")
-            dtype = torch.bfloat16 if dtype is None else dtype
-        else:
-            raise NotImplementedError("FlashDBRX is only available on GPU")
-
-        try:
-            tokenizer = GPT2TokenizerFast.from_pretrained(
-                model_id,
-                revision=revision,
-                padding_side="left",
-                truncation_side="left",
-                trust_remote_code=trust_remote_code,
-                use_fast=True,
-                from_slow=False,
-            )
-        except:
-            try:
-                tokenizer = AutoTokenizer.from_pretrained(
-                    model_id,
-                    revision=revision,
-                    padding_side="left",
-                    truncation_side="left",
-                    trust_remote_code=trust_remote_code,
-                    use_fast=True,
-                    from_slow=False,
-                )
-            except:
-                # FIXME: change back to model id once the tokenizer.json is merged
-                tokenizer = GPT2TokenizerFast.from_pretrained(
-                    "Xenova/dbrx-instruct-tokenizer",
-                    revision=revision,
-                    padding_side="left",
-                    truncation_side="left",
-                    trust_remote_code=trust_remote_code,
-                    use_fast=True,
-                    from_slow=False,
-                )
-
-        config = DbrxConfig.from_pretrained(
-            model_id, revision=revision, trust_remote_code=trust_remote_code
-        )
-        config.quantize = quantize
-        config.speculator = speculator
-
-        torch.distributed.barrier(group=self.process_group)
-
-        filenames = weight_files(model_id, revision=revision, extension=".safetensors")
-        weights = Weights(filenames, device, dtype, process_group=self.process_group)
-        if config.quantize in ["gptq", "awq"]:
-            weights._set_gptq_params(model_id, revision)
-
-        model = FlashDbrxForCausalLM(config, weights)
-
-        torch.distributed.barrier(group=self.process_group)
-        super(FlashDbrx, self).__init__(
-            model=model,
-            tokenizer=tokenizer,
-            num_layers=len(model.model.layers),
-            num_kv_heads=model.model.num_key_value_heads,
-            head_size=model.model.head_size,
-            dtype=dtype,
-            device=device,
-            rank=rank,
-            world_size=world_size,
-        )
diff --git a/server/text_generation_server/models/flash_gemma.py b/server/text_generation_server/models/flash_gemma.py
deleted file mode 100644
index 53bfd064..00000000
--- a/server/text_generation_server/models/flash_gemma.py
+++ /dev/null
@@ -1,74 +0,0 @@
-import torch
-import torch.distributed
-
-from opentelemetry import trace
-from typing import Optional
-from transformers import AutoConfig, AutoTokenizer
-
-from text_generation_server.models import FlashCausalLM
-from text_generation_server.models.custom_modeling.flash_gemma_modeling import (
-    FlashGemmaForCausalLM,
-)
-from text_generation_server.utils import (
-    initialize_torch_distributed,
-    weight_files,
-    Weights,
-)
-
-tracer = trace.get_tracer(__name__)
-
-
-class FlashGemma(FlashCausalLM):
-    def __init__(
-        self,
-        model_id: str,
-        revision: Optional[str] = None,
-        quantize: Optional[str] = None,
-        speculator: Optional[str] = None,
-        dtype: Optional[torch.dtype] = None,
-        trust_remote_code: bool = False,
-    ):
-        self.process_group, rank, world_size = initialize_torch_distributed()
-        if torch.cuda.is_available():
-            device = torch.device(f"cuda:{rank}")
-            dtype = torch.bfloat16 if dtype is None else dtype
-        else:
-            raise NotImplementedError("FlashGemma is only available on GPU")
-
-        tokenizer = AutoTokenizer.from_pretrained(
-            model_id,
-            revision=revision,
-            padding_side="left",
-            truncation_side="left",
-            trust_remote_code=trust_remote_code,
-        )
-
-        config = AutoConfig.from_pretrained(
-            model_id, revision=revision, trust_remote_code=trust_remote_code
-        )
-        config.quantize = quantize
-        config.speculator = speculator
-
-        torch.distributed.barrier(group=self.process_group)
-
-        filenames = weight_files(model_id, revision=revision, extension=".safetensors")
-        weights = Weights(filenames, device, dtype, process_group=self.process_group)
-        if config.quantize in ["gptq", "awq"]:
-            weights._set_gptq_params(model_id, revision)
-
-        # TODO hardcoded
-        prefix = "language_model"
-        model = FlashGemmaForCausalLM(prefix, config, weights, causal=True)
-
-        torch.distributed.barrier(group=self.process_group)
-        super(FlashGemma, self).__init__(
-            model=model,
-            tokenizer=tokenizer,
-            num_layers=len(model.model.layers),
-            num_kv_heads=model.model.num_key_value_heads,
-            head_size=model.model.head_size,
-            dtype=dtype,
-            device=device,
-            rank=rank,
-            world_size=world_size,
-        )
diff --git a/server/text_generation_server/models/flash_gpt2.py b/server/text_generation_server/models/flash_gpt2.py
deleted file mode 100644
index 0067a806..00000000
--- a/server/text_generation_server/models/flash_gpt2.py
+++ /dev/null
@@ -1,77 +0,0 @@
-import torch
-import torch.distributed
-
-from opentelemetry import trace
-from transformers import AutoConfig, AutoTokenizer, GenerationConfig
-from transformers.models.gpt2 import GPT2Tokenizer
-from typing import Optional
-
-from text_generation_server.models import FlashCausalLM
-from text_generation_server.models.custom_modeling.flash_gpt2_modeling import (
-    FlashGPT2ForCausalLM,
-)
-from text_generation_server.utils import (
-    initialize_torch_distributed,
-    weight_files,
-    Weights,
-)
-from text_generation_server.utils.import_utils import SYSTEM
-
-tracer = trace.get_tracer(__name__)
-
-
-class FlashGPT2(FlashCausalLM):
-    def __init__(
-        self,
-        model_id: str,
-        revision: Optional[str] = None,
-        quantize: Optional[str] = None,
-        speculator: Optional[str] = None,
-        dtype: Optional[torch.dtype] = None,
-        trust_remote_code: bool = False,
-    ):
-        self.process_group, rank, world_size = initialize_torch_distributed()
-        if torch.cuda.is_available():
-            device = torch.device(f"cuda:{rank}")
-            dtype = torch.float16 if dtype is None else dtype
-        elif SYSTEM == "xpu":
-            device = torch.device(f"xpu:{rank}")
-            dtype = torch.float16 if dtype is None else dtype
-        else:
-            raise NotImplementedError("FlashGPT2 is only available on GPU")
-
-        tokenizer = AutoTokenizer.from_pretrained(
-            model_id,
-            revision=revision,
-            padding_side="left",
-            truncation_side="left",
-            trust_remote_code=trust_remote_code,
-        )
-
-        config = AutoConfig.from_pretrained(
-            model_id, revision=revision, trust_remote_code=trust_remote_code
-        )
-        config.quantize = quantize
-        config.speculator = speculator
-
-        torch.distributed.barrier(group=self.process_group)
-
-        filenames = weight_files(model_id, revision=revision, extension=".safetensors")
-        weights = Weights(filenames, device, dtype, process_group=self.process_group)
-        if config.quantize in ["gptq", "awq"]:
-            weights._set_gptq_params(model_id, revision)
-
-        prefix = ""
-        model = FlashGPT2ForCausalLM(prefix, config, weights)
-        torch.distributed.barrier(group=self.process_group)
-        super(FlashGPT2, self).__init__(
-            model=model,
-            tokenizer=tokenizer,
-            num_layers=len(model.model.layers),
-            num_kv_heads=model.model.num_heads,
-            head_size=model.model.head_size,
-            dtype=dtype,
-            device=device,
-            rank=rank,
-            world_size=world_size,
-        )
diff --git a/server/text_generation_server/models/flash_llama.py b/server/text_generation_server/models/flash_llama.py
deleted file mode 100644
index 9a7dfaee..00000000
--- a/server/text_generation_server/models/flash_llama.py
+++ /dev/null
@@ -1,86 +0,0 @@
-import torch
-import torch.distributed
-
-from opentelemetry import trace
-from transformers import AutoConfig, AutoTokenizer, GenerationConfig
-from typing import Optional
-
-from text_generation_server.models import FlashCausalLM
-from text_generation_server.models.custom_modeling.flash_llama_modeling import (
-    FlashLlamaForCausalLM,
-)
-from text_generation_server.utils import (
-    initialize_torch_distributed,
-    weight_files,
-    Weights,
-)
-
-tracer = trace.get_tracer(__name__)
-
-from text_generation_server.utils.import_utils import SYSTEM
-
-
-class FlashLlama(FlashCausalLM):
-    def __init__(
-        self,
-        model_id: str,
-        revision: Optional[str] = None,
-        quantize: Optional[str] = None,
-        speculator: Optional[str] = None,
-        dtype: Optional[torch.dtype] = None,
-        trust_remote_code: bool = False,
-    ):
-        self.process_group, rank, world_size = initialize_torch_distributed()
-        if torch.cuda.is_available():
-            device = torch.device(f"cuda:{rank}")
-            dtype = torch.float16 if dtype is None else dtype
-        elif SYSTEM == "xpu":
-            device = torch.device(f"xpu:{rank}")
-            dtype = torch.float16 if dtype is None else dtype
-        else:
-            raise NotImplementedError("FlashLlama is only available on GPU")
-
-        tokenizer = AutoTokenizer.from_pretrained(
-            model_id,
-            revision=revision,
-            padding_side="left",
-            truncation_side="left",
-            trust_remote_code=trust_remote_code,
-        )
-        try:
-            generation_config = GenerationConfig.from_pretrained(
-                model_id, revision=revision, trust_remote_code=trust_remote_code
-            )
-            if isinstance(generation_config.eos_token_id, (list, set)):
-                # TODO Huge hack
-                tokenizer._eos_token_ids = set(generation_config.eos_token_id)
-        except Exception:
-            pass
-
-        config = AutoConfig.from_pretrained(
-            model_id, revision=revision, trust_remote_code=trust_remote_code
-        )
-        config.quantize = quantize
-        config.speculator = speculator
-
-        torch.distributed.barrier(group=self.process_group)
-
-        filenames = weight_files(model_id, revision=revision, extension=".safetensors")
-        weights = Weights(filenames, device, dtype, process_group=self.process_group)
-        if config.quantize in ["gptq", "awq"]:
-            weights._set_gptq_params(model_id, revision)
-
-        prefix = ""
-        model = FlashLlamaForCausalLM(prefix, config, weights)
-        torch.distributed.barrier(group=self.process_group)
-        super(FlashLlama, self).__init__(
-            model=model,
-            tokenizer=tokenizer,
-            num_layers=len(model.model.layers),
-            num_kv_heads=model.model.num_key_value_heads,
-            head_size=model.model.head_size,
-            dtype=dtype,
-            device=device,
-            rank=rank,
-            world_size=world_size,
-        )
diff --git a/server/text_generation_server/models/flash_mistral.py b/server/text_generation_server/models/flash_mistral.py
deleted file mode 100644
index 87e192e0..00000000
--- a/server/text_generation_server/models/flash_mistral.py
+++ /dev/null
@@ -1,608 +0,0 @@
-import math
-import torch
-import torch.distributed
-
-import numpy as np
-
-from dataclasses import dataclass
-from opentelemetry import trace
-from transformers import PreTrainedTokenizerBase, AutoTokenizer, AutoConfig
-from typing import Optional, Tuple, Type
-
-from text_generation_server.pb import generate_pb2
-from text_generation_server.models.flash_causal_lm import FlashCausalLM
-from text_generation_server.models.flash_causal_lm import FlashCausalLMBatch, BLOCK_SIZE
-from text_generation_server.models.cache_manager import (
-    get_cache_manager,
-)
-from text_generation_server.models.custom_modeling.flash_mistral_modeling import (
-    FlashMistralForCausalLM,
-    MistralConfig,
-)
-from text_generation_server.utils.speculate import get_speculate
-from text_generation_server.utils import (
-    initialize_torch_distributed,
-    weight_files,
-    Weights,
-    HeterogeneousNextTokenChooser,
-    StoppingCriteria,
-)
-
-tracer = trace.get_tracer(__name__)
-
-# Will be set in init
-SLIDING_WINDOW: Optional[int] = None
-SLIDING_WINDOW_BLOCKS: Optional[int] = None
-from text_generation_server.utils.import_utils import SYSTEM
-
-MEM_POOL = torch.cuda.graph_pool_handle() if torch.cuda.is_available() else None
-
-
-def set_sliding_window(sliding_window: int, sliding_window_blocks: int):
-    global SLIDING_WINDOW
-    global SLIDING_WINDOW_BLOCKS
-    SLIDING_WINDOW = sliding_window
-    SLIDING_WINDOW_BLOCKS = sliding_window_blocks
-
-
-def get_sliding_windows() -> Tuple[int, int]:
-    global SLIDING_WINDOW
-    global SLIDING_WINDOW_BLOCKS
-    return SLIDING_WINDOW, SLIDING_WINDOW_BLOCKS
-
-
-# Adds windowing logic to FlashCausalLMBatch
-@dataclass
-class FlashMistralBatch(FlashCausalLMBatch):
-    # Prefill cache indices is used to slice into the kv tensor before caching it into the paged attention buffers
-    # as we only keep SLIDING_WINDOW values instead of the whole tensor
-    prefill_cache_indices: Optional[torch.Tensor] = None
-
-    @classmethod
-    def from_pb(
-        cls,
-        pb: generate_pb2.Batch,
-        tokenizer: PreTrainedTokenizerBase,
-        dtype: torch.dtype,
-        device: torch.device,
-    ) -> "FlashCausalLMBatch":
-        batch_tokenized_inputs = cls.batch_tokenized_inputs(pb.requests, tokenizer)
-        return cls.from_tokenized(pb, tokenizer, batch_tokenized_inputs, dtype, device)
-
-    @classmethod
-    def from_tokenized(
-        cls,
-        pb: generate_pb2.Batch,
-        tokenizer: PreTrainedTokenizerBase,
-        batch_tokenized_inputs,
-        dtype: torch.dtype,
-        device: torch.device,
-    ) -> "FlashCausalLMBatch":
-        sliding_window, sliding_window_blocks = get_sliding_windows()
-
-        position_ids = []
-        cu_seqlen_prefill = [0]
-        needed_blocks_slots = []
-        start_slots = []
-        slot_indices = []
-        prefill_cache_indices = []
-
-        input_lengths = []
-        prefix_offsets = []
-        read_offsets = []
-        all_input_ids = []
-        requests_idx_mapping = {}
-
-        all_prefill_logprobs = True
-        no_prefill_logprobs = True
-        prefill_head_indices = []
-        prefill_next_token_indices = []
-        prefill_cu_outlens = [0]
-
-        next_token_chooser_parameters = []
-        stopping_criterias = []
-        top_n_tokens = []
-
-        # Cumulative length
-        cumulative_length = 0
-        cumulative_max_length = 0
-        prefill_out_cumulative_length = 0
-
-        blocks = 0
-        max_seqlen = 0
-        max_length = 0
-        max_blocks = 0
-
-        # Parse batch
-        for i, (r, tokenized_input) in enumerate(
-            zip(pb.requests, batch_tokenized_inputs)
-        ):
-            # request id -> idx in list mapping
-            requests_idx_mapping[r.id] = i
-
-            tokenized_input = tokenized_input[-r.truncate :]
-            if (
-                tokenized_input[0] == tokenizer.bos_token_id
-                and tokenized_input[1] == tokenizer.bos_token_id
-            ):
-                tokenized_input = tokenized_input[1:]
-
-            input_length = len(tokenized_input)
-            input_lengths.append(input_length)
-
-            prefix_offsets.append(input_length - 5)
-            read_offsets.append(input_length)
-
-            all_input_ids.append(tokenized_input)
-
-            # Position ids
-            request_position_ids = torch.arange(0, input_length, dtype=torch.int32)
-            position_ids.append(request_position_ids)
-
-            # Add cumulative lengths of all previous inputs
-            cu_seqlen_prefill.append(cumulative_length + input_length)
-
-            next_token_chooser_parameters.append(r.parameters)
-
-            stopping_criteria = StoppingCriteria.from_pb(
-                r.stopping_parameters, tokenizer
-            )
-            max_new_tokens = stopping_criteria.max_new_tokens
-            stopping_criterias.append(stopping_criteria)
-            top_n_tokens.append(r.top_n_tokens)
-
-            # Paged attention
-            # Remove one as the first token des not have a past
-            speculative_length = get_speculate()
-            total_tokens = input_length + max_new_tokens - 1 + speculative_length
-
-            # Needed blocks can not go over SLIDING_WINDOW_BLOCKS
-            needed_blocks = math.ceil(total_tokens / BLOCK_SIZE)
-            if sliding_window_blocks is not None:
-                needed_blocks = min(needed_blocks, sliding_window_blocks)
-            blocks += needed_blocks
-
-            needed_blocks_slots.append((needed_blocks, total_tokens))
-            start_slots.append(cumulative_max_length)
-
-            request_slot_indices = torch.arange(
-                cumulative_max_length,
-                cumulative_max_length + input_length,
-                dtype=torch.int64,
-            )
-            slot_indices.append(request_slot_indices)
-
-            # Create tensor to slice into the kv tensor in prefill
-            if sliding_window is not None:
-                request_prefill_cache_indices = torch.arange(
-                    cumulative_length + max(0, input_length - sliding_window),
-                    cumulative_length + input_length,
-                    dtype=torch.int64,
-                )
-                prefill_cache_indices.append(request_prefill_cache_indices)
-
-            all_prefill_logprobs = all_prefill_logprobs and r.prefill_logprobs
-            no_prefill_logprobs = no_prefill_logprobs and not r.prefill_logprobs
-
-            if r.prefill_logprobs:
-                prefill_head_indices.append(request_position_ids + cumulative_length)
-                prefill_next_token_indices.append(
-                    prefill_out_cumulative_length + input_length - 1
-                )
-                prefill_cu_outlens.append(prefill_out_cumulative_length + input_length)
-                prefill_out_cumulative_length += input_length
-            else:
-                prefill_head_indices.append(
-                    torch.tensor(
-                        [cumulative_length + input_length - 1], dtype=torch.int32
-                    )
-                )
-                prefill_next_token_indices.append(prefill_out_cumulative_length)
-                prefill_cu_outlens.append(prefill_out_cumulative_length + 1)
-                prefill_out_cumulative_length += 1
-
-            # Update
-            cumulative_length += input_length
-            cumulative_max_length += total_tokens
-            max_seqlen = max(max_seqlen, input_length)
-            max_blocks = max(max_blocks, needed_blocks)
-            max_length = max(
-                max_length, input_length + max_new_tokens + speculative_length
-            )
-
-        next_token_chooser = HeterogeneousNextTokenChooser.from_pb(
-            next_token_chooser_parameters, dtype, device, tokenizer
-        )
-        start_slots = torch.tensor(start_slots, dtype=torch.int64)
-
-        # Padded all_input_ids_tensor
-        all_input_ids_tensor = np.zeros(
-            (len(all_input_ids), max_length), dtype=np.int64
-        )
-        for i, input_ids in enumerate(all_input_ids):
-            all_input_ids_tensor[i, : len(input_ids)] = input_ids
-
-        # Create tensors on device
-        all_input_ids_tensor = torch.tensor(
-            all_input_ids_tensor, dtype=torch.int64, device=device
-        )
-
-        if len(pb.requests) > 1:
-            input_ids = np.concatenate(all_input_ids, dtype=np.int64)
-            position_ids = torch.cat(position_ids)
-            slot_indices = torch.cat(slot_indices)
-            if sliding_window is not None:
-                prefill_cache_indices = torch.cat(prefill_cache_indices)
-        else:
-            input_ids = all_input_ids[0]
-            position_ids = position_ids[0]
-            slot_indices = slot_indices[0]
-            if sliding_window is not None:
-                prefill_cache_indices = prefill_cache_indices[0]
-
-        cu_seqlen_prefill = torch.tensor(
-            cu_seqlen_prefill, device=device, dtype=torch.int32
-        )
-
-        position_ids = position_ids.to(device)
-        slot_indices = slot_indices.to(device)
-        prefill_cache_indices = (
-            prefill_cache_indices.to(device) if sliding_window is not None else None
-        )
-        input_ids = torch.tensor(input_ids, dtype=torch.int64, device=device)
-        input_lengths_tensor = torch.tensor(
-            input_lengths, dtype=torch.int32, device=device
-        )
-
-        if all_prefill_logprobs:
-            prefill_head_indices = None
-            prefill_next_token_indices = cu_seqlen_prefill[1:] - 1
-        elif no_prefill_logprobs:
-            prefill_head_indices = cu_seqlen_prefill[1:] - 1
-            prefill_next_token_indices = None
-        else:
-            prefill_head_indices = torch.tensor(
-                torch.cat(prefill_head_indices), dtype=torch.int64, device=device
-            )
-            prefill_next_token_indices = torch.tensor(
-                prefill_next_token_indices, dtype=torch.int64, device=device
-            )
-        top_n_tokens_tensor = torch.tensor(
-            top_n_tokens, device=device, dtype=torch.int64
-        )
-
-        return cls(
-            batch_id=pb.id,
-            requests=pb.requests,
-            requests_idx_mapping=requests_idx_mapping,
-            input_ids=input_ids,
-            position_ids=position_ids,
-            cu_seqlen_prefill=cu_seqlen_prefill,
-            start_slots=start_slots,
-            slot_indices=slot_indices,
-            needed_blocks_slots=needed_blocks_slots,
-            block_tables=None,
-            block_tables_tensor=None,
-            slots=None,
-            max_seqlen=max_seqlen,
-            prefill_head_indices=prefill_head_indices,
-            prefill_next_token_indices=prefill_next_token_indices,
-            prefill_cu_outlens=prefill_cu_outlens,
-            input_lengths=input_lengths,
-            input_lengths_tensor=input_lengths_tensor,
-            prefix_offsets=prefix_offsets,
-            read_offsets=read_offsets,
-            all_input_ids=all_input_ids,
-            all_input_ids_tensor=all_input_ids_tensor,
-            next_token_chooser=next_token_chooser,
-            stopping_criterias=stopping_criterias,
-            top_n_tokens=top_n_tokens,
-            top_n_tokens_tensor=top_n_tokens_tensor,
-            blocks=blocks,
-            max_blocks=max_blocks,
-            prefill_cache_indices=prefill_cache_indices,
-            speculative_ids=None,
-        )
-
-
-class BaseFlashMistral(FlashCausalLM):
-    def __init__(
-        self,
-        model_cls,
-        model_id: str,
-        config_cls=AutoConfig,
-        revision: Optional[str] = None,
-        quantize: Optional[str] = None,
-        speculator: Optional[str] = None,
-        dtype: Optional[torch.dtype] = None,
-        trust_remote_code: bool = False,
-        tokenizer_class=AutoTokenizer,
-    ):
-        self.process_group, rank, world_size = initialize_torch_distributed()
-        if torch.cuda.is_available():
-            device = torch.device(f"cuda:{rank}")
-            dtype = torch.float16 if dtype is None else dtype
-        elif SYSTEM == "xpu":
-            device = torch.device(f"xpu:{rank}")
-            dtype = torch.float16 if dtype is None else dtype
-        else:
-            raise NotImplementedError("FlashMistral is only available on GPU")
-
-        tokenizer = tokenizer_class.from_pretrained(
-            model_id,
-            revision=revision,
-            padding_side="left",
-            truncation_side="left",
-            trust_remote_code=trust_remote_code,
-        )
-
-        config = config_cls.from_pretrained(
-            model_id, revision=revision, trust_remote_code=trust_remote_code
-        )
-        config.quantize = quantize
-        config.speculator = speculator
-
-        # Set context windows
-        if getattr(config, "sliding_window", None) is not None:
-            set_sliding_window(
-                config.sliding_window, math.ceil(config.sliding_window / BLOCK_SIZE)
-            )
-        else:
-            config.sliding_window = None
-
-        torch.distributed.barrier(group=self.process_group)
-
-        filenames = weight_files(model_id, revision=revision, extension=".safetensors")
-        weights = Weights(filenames, device, dtype, process_group=self.process_group)
-        if config.quantize in ["gptq", "awq"]:
-            weights._set_gptq_params(model_id, revision)
-
-        prefix = ""
-        model = model_cls(prefix, config, weights)
-
-        self.cuda_graphs = {}
-
-        torch.distributed.barrier(group=self.process_group)
-        num_layers, num_kv_heads, head_size = self.get_layer_config(model)
-        super().__init__(
-            model=model,
-            tokenizer=tokenizer,
-            num_layers=num_layers,
-            num_kv_heads=num_kv_heads,
-            head_size=head_size,
-            dtype=dtype,
-            device=device,
-            rank=rank,
-            world_size=world_size,
-            sliding_window=config.sliding_window,
-        )
-
-    def get_layer_config(self, model) -> Tuple[int, int, int]:
-        return (
-            len(model.model.layers),
-            model.model.num_key_value_heads,
-            model.model.head_size,
-        )
-
-    def max_past(self) -> int:
-        return self.model.max_past
-
-    @property
-    def batch_type(self) -> Type[FlashMistralBatch]:
-        return FlashMistralBatch
-
-    def tunableop_warmup(self, seqlen: int):
-        input_ids = torch.zeros(seqlen, dtype=torch.int64, device=self.device)
-        position_ids = torch.zeros(seqlen, dtype=torch.int32, device=self.device)
-        slots = torch.arange(seqlen, dtype=torch.int64, device=self.device)
-        kv_cache = get_cache_manager().kv_cache
-
-        # Dummy value, some models (starcoder2) don't accept `None`.
-        input_lengths = torch.ones(seqlen, dtype=torch.int32, device=self.device)
-
-        # We pass a `cu_seqlen_prefill` in order not to have to deal with paged attention cache allocation/deallocation.
-        self.model.forward(
-            input_ids=input_ids,
-            position_ids=position_ids,
-            cu_seqlen_prefill=torch.tensor(
-                [0, seqlen], device=self.device, dtype=torch.int32
-            ),
-            kv_cache=get_cache_manager().kv_cache,
-            block_tables=None,
-            input_lengths=input_lengths,
-            slots=slots,
-            max_s=seqlen,
-            lm_head_indices=None,
-            prefill_cache_indices=None,
-        )
-
-    def cuda_graph_warmup(self, bs: int, max_s: int, max_bt: int):
-        input_ids = torch.zeros(bs, dtype=torch.int64, device=self.device)
-        position_ids = torch.zeros(bs, dtype=torch.int32, device=self.device)
-        slots = torch.arange(bs, dtype=torch.int64, device=self.device)
-        input_lengths = torch.ones(bs, dtype=torch.int32, device=self.device) * max_s
-        block_tables = (
-            torch.arange(max_bt, dtype=torch.int32, device=self.device)
-            .repeat(bs)
-            .reshape((bs, max_bt))
-        )
-        kv_cache = get_cache_manager().kv_cache
-
-        self.cuda_graphs[bs] = {
-            "input_ids": input_ids,
-            "position_ids": position_ids,
-            "kv_cache": kv_cache,
-            "block_tables": block_tables,
-            "slots": slots,
-            "input_lengths": input_lengths,
-        }
-        graph = torch.cuda.CUDAGraph()
-        self.cuda_graphs[bs]["graph"] = graph
-
-        torch.cuda.synchronize()
-        # Run once outside to warmup
-        self.model.forward(
-            input_ids=input_ids,
-            position_ids=position_ids,
-            cu_seqlen_prefill=None,
-            kv_cache=kv_cache,
-            block_tables=block_tables,
-            slots=slots,
-            input_lengths=input_lengths,
-            max_s=max_s,
-            prefill_cache_indices=None,
-            lm_head_indices=None,
-        )
-        torch.cuda.synchronize()
-
-        with torch.cuda.graph(graph, pool=MEM_POOL):
-            logits, speculative_logits = self.model.forward(
-                input_ids=input_ids,
-                position_ids=position_ids,
-                cu_seqlen_prefill=None,
-                kv_cache=kv_cache,
-                block_tables=block_tables,
-                slots=slots,
-                input_lengths=input_lengths,
-                max_s=max_s,
-                prefill_cache_indices=None,
-                lm_head_indices=None,
-            )
-            self.cuda_graphs[bs]["logits"] = logits
-            self.cuda_graphs[bs]["speculative_logits"] = speculative_logits
-        torch.cuda.synchronize()
-
-    def forward(
-        self, batch: FlashMistralBatch
-    ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
-        # Model Forward
-        if batch.speculative_ids is not None:
-            input_ids = batch.input_ids
-            position_ids = batch.position_ids
-            cu_seqlen_prefill = batch.cu_seqlen_prefill
-            kv_cache = get_cache_manager().kv_cache
-            block_tables = batch.block_tables_tensor
-            slots = batch.slots[batch.slot_indices]
-            input_lengths = batch.input_lengths_tensor
-            max_s = batch.max_seqlen
-            lm_head_indices = batch.prefill_head_indices
-
-            speculative_ids = batch.speculative_ids
-
-            B, speculative_length = speculative_ids.shape
-            new_length = speculative_length + 1
-            new_input_ids = torch.cat(
-                [input_ids.unsqueeze(-1), speculative_ids], dim=1
-            ).reshape(-1)
-            arange = torch.arange(new_length, device=position_ids.device).unsqueeze(0)
-            arange_int = arange.to(dtype=torch.int32)
-            new_position_ids = (
-                position_ids.unsqueeze(-1).expand(B, new_length) + arange
-            ).view(-1)
-            slots = (slots.unsqueeze(-1).expand(B, new_length) + arange_int).view(-1)
-            input_lengths = (
-                input_lengths.unsqueeze(-1).expand(B, new_length) + arange_int
-            ).view(-1)
-
-            # Add Copy the block tables for all members
-            block_tables = (
-                block_tables.unsqueeze(1)
-                .expand(B, new_length, -1)
-                .reshape(B * new_length, -1)
-                .contiguous()
-            )
-            max_s = max_s + speculative_length
-
-            input_ids = new_input_ids
-            position_ids = new_position_ids
-        else:
-            input_ids = batch.input_ids
-            position_ids = batch.position_ids
-            cu_seqlen_prefill = batch.cu_seqlen_prefill
-            kv_cache = get_cache_manager().kv_cache
-            block_tables = batch.block_tables_tensor
-            slots = batch.slots[batch.slot_indices]
-            input_lengths = batch.input_lengths_tensor
-            max_s = batch.max_seqlen
-            lm_head_indices = batch.prefill_head_indices
-
-        if cu_seqlen_prefill is None and self.max_past() is not None:
-            # In decode, not prefill, we're actually overwriting the KV-cache
-            # in a circular buffer mode.
-            # This makes sure the max_s for the decode pass is correct.
-            max_s = min(self.max_past(), max_s)
-
-        bs = input_ids.shape[0]
-        padded_bs = bs
-        if bs == 3:
-            padded_bs = 4
-        elif 3 < bs <= 8:
-            padded_bs = 8
-        elif bs > 8:
-            padded_bs = (bs + 7) // 8 * 8
-
-        # Try to find an associated cuda graph
-        cuda_graph = self.cuda_graphs.get(padded_bs, None)
-
-        if cu_seqlen_prefill is not None or cuda_graph is None:
-            logits, speculative_logits = self.model.forward(
-                input_ids=input_ids,
-                position_ids=position_ids,
-                cu_seqlen_prefill=cu_seqlen_prefill,
-                kv_cache=kv_cache,
-                block_tables=block_tables,
-                slots=slots,
-                input_lengths=input_lengths,
-                max_s=max_s,
-                prefill_cache_indices=batch.prefill_cache_indices,
-                lm_head_indices=lm_head_indices,
-            )
-            if batch.prefill_cache_indices is not None:
-                batch.prefill_cache_indices = None
-            return logits, speculative_logits
-
-        # Copy inputs to the static inputs of the cuda graph
-        # Static inputs are potentially padded
-        cuda_graph["input_ids"][: input_ids.shape[0]] = input_ids
-        cuda_graph["position_ids"][: position_ids.shape[0]] = position_ids
-        cuda_graph["block_tables"][
-            : block_tables.shape[0], : block_tables.shape[1]
-        ] = block_tables
-        cuda_graph["slots"].fill_(-1)
-        cuda_graph["slots"][: slots.shape[0]] = slots
-        cuda_graph["input_lengths"].zero_()
-        cuda_graph["input_lengths"][: input_lengths.shape[0]] = input_lengths
-
-        # Replay the graph
-        cuda_graph["graph"].replay()
-
-        # Slice output to the correct shape
-        speculative_logits = (
-            cuda_graph["speculative_logits"][:bs]
-            if cuda_graph["speculative_logits"] is not None
-            else None
-        )
-        logits = cuda_graph["logits"][:bs]
-        return logits, speculative_logits
-
-
-class FlashMistral(BaseFlashMistral):
-    def __init__(
-        self,
-        model_id: str,
-        revision: Optional[str] = None,
-        quantize: Optional[str] = None,
-        speculator: Optional[str] = None,
-        dtype: Optional[torch.dtype] = None,
-        trust_remote_code: bool = False,
-    ):
-        super(FlashMistral, self).__init__(
-            config_cls=MistralConfig,
-            model_cls=FlashMistralForCausalLM,
-            model_id=model_id,
-            revision=revision,
-            quantize=quantize,
-            speculator=speculator,
-            dtype=dtype,
-            trust_remote_code=trust_remote_code,
-        )
diff --git a/server/text_generation_server/models/flash_mixtral.py b/server/text_generation_server/models/flash_mixtral.py
deleted file mode 100644
index 587d423f..00000000
--- a/server/text_generation_server/models/flash_mixtral.py
+++ /dev/null
@@ -1,31 +0,0 @@
-import torch
-
-from typing import Optional
-
-from text_generation_server.models.flash_mistral import BaseFlashMistral
-from text_generation_server.models.custom_modeling.flash_mixtral_modeling import (
-    MixtralConfig,
-    FlashMixtralForCausalLM,
-)
-
-
-class FlashMixtral(BaseFlashMistral):
-    def __init__(
-        self,
-        model_id: str,
-        revision: Optional[str] = None,
-        quantize: Optional[str] = None,
-        speculator: Optional[str] = None,
-        dtype: Optional[torch.dtype] = None,
-        trust_remote_code: bool = False,
-    ):
-        super(FlashMixtral, self).__init__(
-            config_cls=MixtralConfig,
-            model_cls=FlashMixtralForCausalLM,
-            model_id=model_id,
-            revision=revision,
-            quantize=quantize,
-            speculator=speculator,
-            dtype=dtype,
-            trust_remote_code=trust_remote_code,
-        )
diff --git a/server/text_generation_server/models/flash_neox.py b/server/text_generation_server/models/flash_neox.py
deleted file mode 100644
index adefaeb2..00000000
--- a/server/text_generation_server/models/flash_neox.py
+++ /dev/null
@@ -1,77 +0,0 @@
-import torch
-import torch.distributed
-
-from opentelemetry import trace
-from transformers import AutoTokenizer, AutoConfig
-from typing import Optional
-
-from text_generation_server.models import FlashCausalLM
-from text_generation_server.models.custom_modeling.flash_neox_modeling import (
-    FlashGPTNeoXForCausalLM,
-)
-from text_generation_server.utils import (
-    initialize_torch_distributed,
-    weight_files,
-    Weights,
-)
-from text_generation_server.utils.import_utils import SYSTEM
-
-tracer = trace.get_tracer(__name__)
-
-
-class FlashNeoXSharded(FlashCausalLM):
-    def __init__(
-        self,
-        model_id: str,
-        revision: Optional[str] = None,
-        quantize: Optional[str] = None,
-        speculator: Optional[str] = None,
-        dtype: Optional[torch.dtype] = None,
-        trust_remote_code: bool = False,
-    ):
-        self.process_group, rank, world_size = initialize_torch_distributed()
-        if torch.cuda.is_available():
-            device = torch.device(f"cuda:{rank}")
-            dtype = torch.float16 if dtype is None else dtype
-        elif SYSTEM == "xpu":
-            device = torch.device(f"xpu:{rank}")
-            dtype = torch.float16 if dtype is None else dtype
-        else:
-            raise NotImplementedError("FlashNeoX is only available on GPU")
-
-        tokenizer = AutoTokenizer.from_pretrained(
-            model_id,
-            revision=revision,
-            padding_side="left",
-            truncation_side="left",
-            trust_remote_code=trust_remote_code,
-        )
-
-        config = AutoConfig.from_pretrained(
-            model_id, revision=revision, trust_remote_code=trust_remote_code
-        )
-        config.quantize = quantize
-        config.speculator = speculator
-
-        torch.distributed.barrier(group=self.process_group)
-        filenames = weight_files(model_id, revision=revision, extension=".safetensors")
-        weights = Weights(
-            filenames, device=device, dtype=dtype, process_group=self.process_group
-        )
-        if config.quantize == "gptq":
-            weights._set_gptq_params(model_id, revision)
-
-        model = FlashGPTNeoXForCausalLM(config, weights)
-
-        torch.distributed.barrier(group=self.process_group)
-        super(FlashNeoXSharded, self).__init__(
-            model=model.to(device),
-            tokenizer=tokenizer,
-            num_layers=len(model.gpt_neox.layers),
-            num_kv_heads=model.gpt_neox.num_heads,
-            head_size=model.gpt_neox.head_size,
-            dtype=dtype,
-            device=device,
-            rank=rank,
-            world_size=world_size,
-        )
diff --git a/server/text_generation_server/models/flash_phi.py b/server/text_generation_server/models/flash_phi.py
deleted file mode 100644
index 32b573a9..00000000
--- a/server/text_generation_server/models/flash_phi.py
+++ /dev/null
@@ -1,103 +0,0 @@
-import torch
-import torch.distributed
-
-from opentelemetry import trace
-from transformers import AutoConfig, AutoTokenizer
-from typing import Optional
-
-from text_generation_server.models import FlashCausalLM
-from text_generation_server.models.custom_modeling.flash_phi_modeling import (
-    FlashPhiForCausalLM,
-    PhiConfig,
-)
-from text_generation_server.utils import (
-    initialize_torch_distributed,
-    weight_files,
-    Weights,
-)
-
-tracer = trace.get_tracer(__name__)
-
-
-class FlashPhi(FlashCausalLM):
-    def __init__(
-        self,
-        model_id: str,
-        revision: Optional[str] = None,
-        quantize: Optional[str] = None,
-        speculator: Optional[str] = None,
-        dtype: Optional[torch.dtype] = None,
-        trust_remote_code: bool = False,
-    ):
-        self.process_group, rank, world_size = initialize_torch_distributed()
-        if torch.cuda.is_available():
-            device = torch.device(f"cuda:{rank}")
-            dtype = torch.float16 if dtype is None else dtype
-        else:
-            raise NotImplementedError("FlashPhi is only available on GPU")
-
-        tokenizer = AutoTokenizer.from_pretrained(
-            model_id,
-            revision=revision,
-            padding_side="left",
-            truncation_side="left",
-            trust_remote_code=trust_remote_code,
-        )
-
-        config = PhiConfig.from_pretrained(
-            model_id, revision=revision, trust_remote_code=trust_remote_code
-        )
-        config.quantize = quantize
-        config.speculator = speculator
-
-        torch.distributed.barrier(group=self.process_group)
-
-        filenames = weight_files(model_id, revision=revision, extension=".safetensors")
-        weights = Weights(filenames, device, dtype, process_group=self.process_group)
-        if config.quantize in ["gptq", "awq"]:
-            weights._set_gptq_params(model_id, revision)
-
-        model = FlashPhiForCausalLM(config, weights)
-        if speculator:
-            from text_generation_server.utils.medusa import MedusaModel
-            from huggingface_hub import hf_hub_download
-            import json
-            import os
-            from pathlib import Path
-
-            is_local_model = (
-                Path(speculator).exists() and Path(speculator).is_dir()
-            ) or os.getenv("WEIGHTS_CACHE_OVERRIDE", None) is not None
-
-            if not is_local_model:
-                medusa_config = hf_hub_download(
-                    speculator, revision=revision, filename="config.json"
-                )
-                medusa_head = hf_hub_download(
-                    speculator, revision=revision, filename="medusa_lm_head.pt"
-                )
-            else:
-                medusa_config = str(Path(speculator) / "config.json")
-                medusa_head = str(Path(speculator) / "medusa_lm_head.pt")
-
-            with open(medusa_config, "r") as f:
-                config = json.load(f)
-            medusa_sf = medusa_head[: -len(".pt")] + ".safetensors"
-            weights = Weights(
-                [medusa_sf], device, dtype, process_group=self.process_group
-            )
-            lm_head = model.lm_head
-            model.lm_head = MedusaModel(config, weights, lm_head)
-
-        torch.distributed.barrier(group=self.process_group)
-        super(FlashPhi, self).__init__(
-            model=model,
-            tokenizer=tokenizer,
-            num_layers=len(model.model.layers),
-            num_kv_heads=model.model.num_key_value_heads,
-            head_size=model.model.head_size,
-            dtype=dtype,
-            device=device,
-            rank=rank,
-            world_size=world_size,
-        )
diff --git a/server/text_generation_server/models/flash_qwen2.py b/server/text_generation_server/models/flash_qwen2.py
deleted file mode 100644
index 59064b30..00000000
--- a/server/text_generation_server/models/flash_qwen2.py
+++ /dev/null
@@ -1,87 +0,0 @@
-import math
-
-import torch
-import torch.distributed
-
-from opentelemetry import trace
-from transformers import AutoTokenizer, AutoConfig
-from typing import Optional
-
-from text_generation_server.models.cache_manager import BLOCK_SIZE
-from text_generation_server.models.flash_mistral import (
-    BaseFlashMistral,
-    set_sliding_window,
-)
-from text_generation_server.models.custom_modeling.flash_qwen2_modeling import (
-    Qwen2ForCausalLM,
-)
-from text_generation_server.utils import (
-    initialize_torch_distributed,
-    weight_files,
-    Weights,
-)
-
-tracer = trace.get_tracer(__name__)
-
-
-class FlashQwen2(BaseFlashMistral):
-    def __init__(
-        self,
-        model_id: str,
-        revision: Optional[str] = None,
-        quantize: Optional[str] = None,
-        speculator: Optional[str] = None,
-        dtype: Optional[torch.dtype] = None,
-        trust_remote_code: bool = False,
-    ):
-        self.process_group, rank, world_size = initialize_torch_distributed()
-        if torch.cuda.is_available():
-            device = torch.device(f"cuda:{rank}")
-            dtype = torch.float16 if dtype is None else dtype
-        else:
-            raise NotImplementedError("FlashQwen2 is only available on GPU")
-
-        tokenizer = AutoTokenizer.from_pretrained(
-            model_id,
-            revision=revision,
-            padding_side="left",
-            truncation_side="left",
-            trust_remote_code=trust_remote_code,
-        )
-
-        config = AutoConfig.from_pretrained(
-            model_id, revision=revision, trust_remote_code=trust_remote_code
-        )
-        config.quantize = quantize
-        config.speculator = speculator
-
-        # Set context windows
-        if config.sliding_window is not None:
-            set_sliding_window(
-                config.sliding_window, math.ceil(config.sliding_window / BLOCK_SIZE)
-            )
-
-        torch.distributed.barrier(group=self.process_group)
-
-        filenames = weight_files(model_id, revision=revision, extension=".safetensors")
-        weights = Weights(filenames, device, dtype, process_group=self.process_group)
-        if config.quantize in ["gptq", "awq"]:
-            weights._set_gptq_params(model_id, revision)
-
-        model = Qwen2ForCausalLM(config, weights)
-
-        self.cuda_graphs = {}
-
-        torch.distributed.barrier(group=self.process_group)
-        super(BaseFlashMistral, self).__init__(
-            model=model,
-            tokenizer=tokenizer,
-            num_layers=len(model.model.layers),
-            num_kv_heads=model.model.num_key_value_heads,
-            head_size=model.model.head_size,
-            dtype=dtype,
-            device=device,
-            rank=rank,
-            world_size=world_size,
-            sliding_window=config.sliding_window,
-        )
diff --git a/server/text_generation_server/models/flash_rw.py b/server/text_generation_server/models/flash_rw.py
deleted file mode 100644
index e6350611..00000000
--- a/server/text_generation_server/models/flash_rw.py
+++ /dev/null
@@ -1,86 +0,0 @@
-import torch
-import torch.distributed
-
-from opentelemetry import trace
-from transformers import AutoTokenizer
-from typing import Optional
-
-from text_generation_server.models import FlashCausalLM
-from text_generation_server.models.custom_modeling.flash_rw_modeling import (
-    RWConfig,
-    FlashRWForCausalLM,
-)
-from text_generation_server.utils import (
-    initialize_torch_distributed,
-    weight_files,
-    Weights,
-)
-from text_generation_server.utils.import_utils import SYSTEM
-
-tracer = trace.get_tracer(__name__)
-
-
-class FlashRWSharded(FlashCausalLM):
-    def __init__(
-        self,
-        model_id: str,
-        revision: Optional[str] = None,
-        quantize: Optional[str] = None,
-        speculator: Optional[str] = None,
-        dtype: Optional[torch.dtype] = None,
-        trust_remote_code: bool = False,
-    ):
-        self.process_group, rank, world_size = initialize_torch_distributed()
-        if torch.cuda.is_available():
-            device = torch.device(f"cuda:{rank}")
-            dtype = torch.float16 if dtype is None else dtype
-        elif SYSTEM == "xpu":
-            device = torch.device(f"xpu:{rank}")
-            dtype = torch.float16 if dtype is None else dtype
-        else:
-            raise NotImplementedError("FlashRW is only available on GPU")
-
-        tokenizer = AutoTokenizer.from_pretrained(
-            model_id,
-            revision=revision,
-            padding_side="left",
-            truncation_side="left",
-            trust_remote_code=trust_remote_code,
-        )
-
-        config = RWConfig.from_pretrained(
-            model_id, revision=revision, trust_remote_code=trust_remote_code
-        )
-
-        torch.distributed.barrier(group=self.process_group)
-        filenames = weight_files(model_id, revision=revision, extension=".safetensors")
-        weights = Weights(
-            filenames,
-            device,
-            dtype,
-            process_group=self.process_group,
-            aliases={
-                "lm_head.weight": ["transformer.word_embeddings.weight"],
-                "transformer.word_embeddings.weight": ["lm_head.weight"],
-            },
-        )
-
-        config.quantize = quantize
-        config.speculator = speculator
-        if config.quantize == "gptq":
-            weights._set_gptq_params(model_id, revision)
-
-        model = FlashRWForCausalLM(config, weights)
-
-        torch.distributed.barrier(group=self.process_group)
-        super(FlashRWSharded, self).__init__(
-            model=model.to(device),
-            tokenizer=tokenizer,
-            num_layers=len(model.transformer.h),
-            num_kv_heads=model.transformer.cache_size,
-            head_size=model.transformer.head_size,
-            dtype=dtype,
-            device=device,
-            rank=rank,
-            world_size=world_size,
-        )
diff --git a/server/text_generation_server/models/flash_santacoder.py b/server/text_generation_server/models/flash_santacoder.py
deleted file mode 100644
index 2ad36b93..00000000
--- a/server/text_generation_server/models/flash_santacoder.py
+++ /dev/null
@@ -1,94 +0,0 @@
-import torch
-import torch.distributed
-
-from opentelemetry import trace
-from transformers import AutoTokenizer, AutoConfig
-from typing import Optional, List
-import json
-import os
-
-from huggingface_hub import hf_hub_download
-from text_generation_server.models import FlashCausalLM
-from text_generation_server.models.custom_modeling.flash_santacoder_modeling import (
-    FlashSantacoderForCausalLM,
-)
-from text_generation_server.utils import (
-    initialize_torch_distributed,
-    weight_files,
-    Weights,
-)
-
-from text_generation_server.utils.import_utils import SYSTEM
-
-tracer = trace.get_tracer(__name__)
-
-
-class FlashSantacoderSharded(FlashCausalLM):
-    def __init__(
-        self,
-        model_id: str,
-        revision: Optional[str] = None,
-        quantize: Optional[str] = None,
-        speculator: Optional[str] = None,
-        dtype: Optional[torch.dtype] = None,
-        trust_remote_code: bool = False,
-    ):
-        self.process_group, rank, world_size = initialize_torch_distributed()
-        if torch.cuda.is_available():
-            device = torch.device(f"cuda:{rank}")
-            dtype = torch.float16 if dtype is None else dtype
-        elif SYSTEM == "xpu":
-            device = torch.device(f"xpu:{rank}")
-            dtype = torch.float16 if dtype is None else dtype
-        else:
-            raise NotImplementedError("FlashSantacoderSharded is only available on GPU")
-
-        tokenizer = AutoTokenizer.from_pretrained(
-            model_id,
-            revision=revision,
-            padding_side="left",
-            truncation_side="left",
-            trust_remote_code=trust_remote_code,
-        )
-
-        config = AutoConfig.from_pretrained(
-            model_id,
-            revision=revision,
-            trust_remote_code=True,
-        )
-        config.quantize = quantize
-        config.speculator = speculator
-        config.transpose = config.architectures[0].startswith("GPT2")
-
-        torch.distributed.barrier(group=self.process_group)
-        filenames = weight_files(model_id, revision=revision, extension=".safetensors")
-        weights = Weights(
-            filenames,
-            device=device,
-            dtype=dtype,
-            process_group=self.process_group,
-            aliases={"transformer.wte.weight": ["lm_head.weight"]},
-        )
-        if config.quantize == "gptq":
-            weights._set_gptq_params(model_id, revision)
-
-        model = FlashSantacoderForCausalLM(config, weights)
-
-        torch.distributed.barrier(group=self.process_group)
-        super(FlashSantacoderSharded, self).__init__(
-            model=model.to(device),
-            tokenizer=tokenizer,
-            num_layers=len(model.transformer.h),
-            num_kv_heads=1,
-            head_size=model.transformer.head_size,
-            dtype=dtype,
-            device=device,
-            rank=rank,
-            world_size=world_size,
-        )
-
-    def decode(self, generated_ids: List[int]) -> str:
-        # Do not skip special tokens as they are used for custom parsing rules of the generated text
-        return self.tokenizer.decode(
-            generated_ids, skip_special_tokens=False, clean_up_tokenization_spaces=False
-        )
diff --git a/server/text_generation_server/models/flash_starcoder2.py b/server/text_generation_server/models/flash_starcoder2.py
deleted file mode 100644
index dc5d49be..00000000
--- a/server/text_generation_server/models/flash_starcoder2.py
+++ /dev/null
@@ -1,86 +0,0 @@
-import math
-
-import torch
-
-from typing import Optional
-
-from transformers.models.gpt2 import GPT2TokenizerFast
-
-from text_generation_server.models.cache_manager import BLOCK_SIZE
-from text_generation_server.models.flash_mistral import (
-    BaseFlashMistral,
-    set_sliding_window,
-)
-from text_generation_server.models.custom_modeling.flash_starcoder2_modeling import (
-    Starcoder2Config,
-    FlashStarcoder2ForCausalLM,
-)
-from text_generation_server.utils import (
-    initialize_torch_distributed,
-    weight_files,
-    Weights,
-)
-
-
-# Starcoder2 has the same base as Mistral
-class FlashStarcoder2(BaseFlashMistral):
-    def __init__(
-        self,
-        model_id: str,
-        revision: Optional[str] = None,
-        quantize: Optional[str] = None,
-        speculator: Optional[str] = None,
-        dtype: Optional[torch.dtype] = None,
-        trust_remote_code: bool = False,
-    ):
-        self.process_group, rank, world_size = initialize_torch_distributed()
-        if torch.cuda.is_available():
-            device = torch.device(f"cuda:{rank}")
-            dtype = torch.float16 if dtype is None else dtype
-        else:
-            raise NotImplementedError("FlashStarcoder2 is only available on GPU")
-
-        tokenizer = GPT2TokenizerFast.from_pretrained(
-            model_id,
-            revision=revision,
-            padding_side="left",
-            truncation_side="left",
-            trust_remote_code=trust_remote_code,
-        )
-
-        config = Starcoder2Config.from_pretrained(
-            model_id, revision=revision, trust_remote_code=trust_remote_code
-        )
-        config.quantize = quantize
-        config.speculator = speculator
-
-        # Set context windows
-        if config.sliding_window is not None:
-            set_sliding_window(
-                config.sliding_window, math.ceil(config.sliding_window / BLOCK_SIZE)
-            )
-
-        torch.distributed.barrier(group=self.process_group)
-
-        filenames = weight_files(model_id, revision=revision, extension=".safetensors")
-        weights = Weights(filenames, device, dtype, process_group=self.process_group)
-        if config.quantize in ["gptq", "awq"]:
-            weights._set_gptq_params(model_id, revision)
-
-        model = FlashStarcoder2ForCausalLM(config, weights)
-
-        self.cuda_graphs = {}
-
-        torch.distributed.barrier(group=self.process_group)
-        super(BaseFlashMistral, self).__init__(
-            model=model,
-            tokenizer=tokenizer,
-            num_layers=len(model.model.layers),
-            num_kv_heads=model.model.num_key_value_heads,
-            head_size=model.model.head_size,
-            dtype=dtype,
-            device=device,
-            rank=rank,
-            world_size=world_size,
-            sliding_window=config.sliding_window,
-        )
diff --git a/server/text_generation_server/models/galactica.py b/server/text_generation_server/models/galactica.py
index 4656fd45..7c4e462c 100644
--- a/server/text_generation_server/models/galactica.py
+++ b/server/text_generation_server/models/galactica.py
@@ -2,24 +2,17 @@ import re
 import torch
 import torch.distributed
 
-from typing import List, Optional, Type
 
 from transformers import (
-    AutoTokenizer,
-    AutoConfig,
     PreTrainedTokenizerBase,
 )
-from text_generation_server.models import CausalLM
 from text_generation_server.models.causal_lm import CausalLMBatch
 from text_generation_server.pb import generate_pb2
-from text_generation_server.models.custom_modeling.opt_modeling import OPTForCausalLM
 from text_generation_server.utils import (
     NextTokenChooser,
     StoppingCriteria,
-    initialize_torch_distributed,
-    weight_files,
-    Weights,
 )
+from text_generation_server.utils.chunks import concat_text_chunks
 
 # CREDIT: Papers with code => https://github.com/paperswithcode/galai/blob/main/galai/utils.py
 
@@ -91,7 +84,9 @@ class GalacticaCausalLMBatch(CausalLMBatch):
         for i, r in enumerate(pb.requests):
             requests_idx_mapping[r.id] = i
             # Add escape_custom_split_sequence to the CausalLMBatch logic
-            inputs.append(escape_custom_split_sequence(r.inputs))
+            inputs.append(
+                escape_custom_split_sequence(concat_text_chunks(r.input_chunks.chunks))
+            )
             next_token_choosers.append(
                 NextTokenChooser.from_pb(r.parameters, device, tokenizer)
             )
@@ -159,82 +154,3 @@ class GalacticaCausalLMBatch(CausalLMBatch):
             padding_right_offset=padding_right_offset,
             max_tokens=max_tokens,
         )
-
-
-class GalacticaSharded(CausalLM):
-    def __init__(
-        self,
-        model_id: str,
-        revision: Optional[str] = None,
-        quantize: Optional[str] = None,
-        speculator: Optional[str] = None,
-        dtype: Optional[torch.dtype] = None,
-        trust_remote_code: bool = False,
-    ):
-        self.process_group, rank, world_size = initialize_torch_distributed()
-        if torch.cuda.is_available():
-            device = torch.device(f"cuda:{rank}")
-            dtype = torch.float16 if dtype is None else dtype
-        else:
-            device = torch.device("cpu")
-            dtype = torch.float32 if dtype is None else dtype
-
-        tokenizer = AutoTokenizer.from_pretrained(
-            model_id,
-            revision=revision,
-            padding_side="left",
-            truncation_side="left",
-            trust_remote_code=trust_remote_code,
-        )
-
-        config = AutoConfig.from_pretrained(
-            model_id,
-            revision=revision,
-            tp_parallel=True,
-            trust_remote_code=trust_remote_code,
-        )
-        config.quantize = quantize
-        tokenizer.pad_token_id = config.pad_token_id
-        config.speculator = speculator
-
-        torch.distributed.barrier(group=self.process_group)
-        filenames = weight_files(model_id, revision=revision, extension=".safetensors")
-        weights = Weights(
-            filenames, device=device, dtype=dtype, process_group=self.process_group
-        )
-        if config.quantize == "gptq":
-            weights._set_gptq_params(model_id, revision)
-
-        model = OPTForCausalLM(config, weights)
-
-        torch.distributed.barrier(group=self.process_group)
-        super(CausalLM, self).__init__(
-            model=model,
-            tokenizer=tokenizer,
-            requires_padding=True,
-            dtype=dtype,
-            device=device,
-            rank=rank,
-            world_size=world_size,
-        )
-
-    @property
-    def batch_type(self) -> Type[CausalLMBatch]:
-        return GalacticaCausalLMBatch
-
-    def decode(self, generated_ids: List[int]) -> str:
-        # Do not skip special tokens as they are used for custom parsing rules of the generated text
-        return self.tokenizer.decode(
-            generated_ids, skip_special_tokens=False, clean_up_tokenization_spaces=False
-        )
-
-    def forward(
-        self, input_ids, attention_mask, position_ids, past_key_values: Optional = None
-    ):
-        outputs, speculative_logits = self.model.forward(
-            input_ids=input_ids,
-            attention_mask=attention_mask,
-            past_key_values=past_key_values,
-            use_cache=True,
-        )
-        return outputs.logits, speculative_logits, outputs.past_key_values
diff --git a/server/text_generation_server/models/globals.py b/server/text_generation_server/models/globals.py
index e8a11958..8ef8ec82 100644
--- a/server/text_generation_server/models/globals.py
+++ b/server/text_generation_server/models/globals.py
@@ -1,7 +1,40 @@
 import torch
 import os
+from typing import Dict, Optional
+from loguru import logger
+from text_generation_server.utils.log import log_master
+
+ATTENTION = os.environ["ATTENTION"]
+# default_prefix_caching = "1" if ATTENTION in {"flashinfer", "flashdecoding"} else "0"
+PREFIX_CACHING = os.environ["PREFIX_CACHING"].lower() in {
+    "1",
+    "true",
+}
+PREFILL_CHUNKING = os.getenv("PREFILL_CHUNKING", "0").lower() in {"1", "true"}
+log_master(logger.info, f"Using prefix caching = {PREFIX_CACHING}")
+_expected = {"paged", "flashdecoding", "flashinfer", "default"}
+assert (
+    ATTENTION in _expected
+), f"Attention is not valid {ATTENTION}, expected {_expected}"
+log_master(logger.info, f"Using Attention = {ATTENTION}")
+
+if PREFIX_CACHING and ATTENTION not in {"flashinfer", "flashdecoding"}:
+    raise RuntimeError("Prefix caching is only supported with flashinfer")
 
 MEM_POOL = torch.cuda.graph_pool_handle() if torch.cuda.is_available() else None
+TGI_WIGGLE_ROOM = float(os.getenv("TGI_WIGGLE_ROOM", "0.90"))
+assert TGI_WIGGLE_ROOM > 0
+assert TGI_WIGGLE_ROOM < 1
+
+# This is overridden by the cli
+BLOCK_SIZE: int
+if ATTENTION == "flashdecoding":
+    BLOCK_SIZE = 256
+elif ATTENTION == "flashinfer":
+    BLOCK_SIZE = 1
+else:
+    BLOCK_SIZE = 16
+
 # This is overridden by the cli
 cuda_graphs = os.getenv("CUDA_GRAPHS")
 if cuda_graphs is not None:
@@ -24,3 +57,17 @@ MODEL_ID = None
 def set_model_id(model_id: str):
     global MODEL_ID
     MODEL_ID = model_id
+
+# NOTE: eventually we should move this into the router and pass back the
+# index in all cases.
+ADAPTER_TO_INDEX: Optional[Dict[str, int]] = None
+
+
+def set_adapter_to_index(adapter_to_index: Dict[str, int]):
+    global ADAPTER_TO_INDEX
+    ADAPTER_TO_INDEX = adapter_to_index
+
+
+def get_adapter_to_index():
+    global ADAPTER_TO_INDEX
+    return ADAPTER_TO_INDEX
diff --git a/server/text_generation_server/models/gpt_neox.py b/server/text_generation_server/models/gpt_neox.py
deleted file mode 100644
index c0e1adf2..00000000
--- a/server/text_generation_server/models/gpt_neox.py
+++ /dev/null
@@ -1,89 +0,0 @@
-import torch
-import torch.distributed
-
-from typing import Optional
-
-from transformers import (
-    AutoTokenizer,
-    AutoConfig,
-)
-from text_generation_server.models import CausalLM
-from text_generation_server.models.custom_modeling.neox_modeling import (
-    GPTNeoxForCausalLM,
-)
-from text_generation_server.utils import (
-    initialize_torch_distributed,
-    weight_files,
-    Weights,
-)
-
-
-class GPTNeoxSharded(CausalLM):
-    def __init__(
-        self,
-        model_id: str,
-        revision: Optional[str] = None,
-        quantize: Optional[str] = None,
-        speculator: Optional[str] = None,
-        dtype: Optional[torch.dtype] = None,
-        trust_remote_code: bool = False,
-    ):
-        self.process_group, rank, world_size = initialize_torch_distributed()
-        if torch.cuda.is_available():
-            device = torch.device(f"cuda:{rank}")
-            dtype = torch.float16 if dtype is None else dtype
-        else:
-            device = torch.device("cpu")
-            dtype = torch.float32 if dtype is None else dtype
-
-        tokenizer = AutoTokenizer.from_pretrained(
-            model_id,
-            revision=revision,
-            padding_side="left",
-            truncation_side="left",
-            trust_remote_code=trust_remote_code,
-        )
-        tokenizer.pad_token = tokenizer.eos_token
-
-        config = AutoConfig.from_pretrained(
-            model_id,
-            revision=revision,
-            trust_remote_code=trust_remote_code,
-        )
-        config.quantize = quantize
-        config.speculator = speculator
-
-        torch.distributed.barrier(group=self.process_group)
-        filenames = weight_files(model_id, revision=revision, extension=".safetensors")
-        weights = Weights(
-            filenames, device=device, dtype=dtype, process_group=self.process_group
-        )
-        if config.quantize == "gptq":
-            weights._set_gptq_params(model_id, revision)
-
-        model = GPTNeoxForCausalLM(config, weights)
-
-        torch.distributed.barrier(group=self.process_group)
-        super(CausalLM, self).__init__(
-            model=model,
-            tokenizer=tokenizer,
-            requires_padding=True,
-            dtype=dtype,
-            device=device,
-            rank=rank,
-            world_size=world_size,
-        )
-
-    def forward(
-        self, input_ids, attention_mask, position_ids, past_key_values: Optional = None
-    ):
-        outputs, speculative_logits = self.model.forward(
-            input_ids=input_ids,
-            attention_mask=attention_mask,
-            position_ids=position_ids,
-            past_key_values=past_key_values,
-            use_cache=True,
-        )
-
-        logits = outputs.logits
-        return logits, speculative_logits, outputs.past_key_values
diff --git a/server/text_generation_server/models/idefics.py b/server/text_generation_server/models/idefics.py
deleted file mode 100644
index c1fe03e4..00000000
--- a/server/text_generation_server/models/idefics.py
+++ /dev/null
@@ -1,93 +0,0 @@
-import torch
-import torch.distributed
-
-from typing import List, Optional, Tuple
-
-from transformers import (
-    AutoTokenizer,
-    AutoConfig,
-    AutoProcessor,
-)
-
-from text_generation_server.models.custom_modeling.idefics_config import IdeficsConfig
-from text_generation_server.models.custom_modeling.idefics_processing import (
-    IdeficsProcessor,
-)
-from transformers import LlamaTokenizerFast
-from text_generation_server.models.custom_modeling.idefics_modeling import (
-    IdeficsForVisionText2Text,
-)
-from text_generation_server.models.idefics_causal_lm import IdeficsCausalLM
-from text_generation_server.utils import (
-    initialize_torch_distributed,
-    weight_files,
-    Weights,
-)
-
-
-class IDEFICSSharded(IdeficsCausalLM):
-    def __init__(
-        self,
-        model_id: str,
-        revision: Optional[str] = None,
-        quantize: Optional[str] = None,
-        speculator: Optional[str] = None,
-        dtype: Optional[torch.dtype] = None,
-        trust_remote_code: bool = False,
-    ):
-        self.process_group, rank, world_size = initialize_torch_distributed()
-        if torch.cuda.is_available():
-            device = torch.device(f"cuda:{rank}")
-            # 9b seems to work correctly enough in float16, but 80b seems
-            # to be really saturating for f16.
-            dtype = torch.float16 if dtype is None else dtype
-        else:
-            device = torch.device("cpu")
-            dtype = torch.float32 if dtype is None else dtype
-        self.device, self.dtype = device, dtype
-
-        config = IdeficsConfig.from_pretrained(
-            model_id,
-            revision=revision,
-            trust_remote_code=trust_remote_code,
-        )
-        config.quantize = quantize
-        config.speculator = speculator
-        config.vision_config.quantize = quantize
-
-        tokenizer = LlamaTokenizerFast.from_pretrained(
-            model_id,
-            revision=revision,
-            padding_side="left",
-            truncation_side="left",
-            trust_remote_code=trust_remote_code,
-        )
-        self.processor = IdeficsProcessor.from_pretrained(
-            model_id,
-            revision=revision,
-            padding_side="left",
-            truncation_side="left",
-            trust_remote_code=trust_remote_code,
-        )
-
-        torch.distributed.barrier(group=self.process_group)
-        filenames = weight_files(model_id, revision=revision, extension=".safetensors")
-        weights = Weights(
-            filenames,
-            device=device,
-            dtype=dtype,
-            process_group=self.process_group,
-        )
-
-        model = IdeficsForVisionText2Text(config, weights)
-
-        torch.distributed.barrier(group=self.process_group)
-        super(IdeficsCausalLM, self).__init__(
-            model=model,
-            tokenizer=tokenizer,
-            requires_padding=True,
-            dtype=dtype,
-            device=device,
-            rank=rank,
-            world_size=world_size,
-        )
diff --git a/server/text_generation_server/models/idefics2.py b/server/text_generation_server/models/idefics2.py
deleted file mode 100644
index 314c0500..00000000
--- a/server/text_generation_server/models/idefics2.py
+++ /dev/null
@@ -1,51 +0,0 @@
-import torch
-
-from typing import Optional, Tuple
-
-from transformers import (
-    AutoProcessor,
-)
-from text_generation_server.models.custom_modeling.idefics2 import (
-    Idefics2ForConditionalGeneration,
-)
-
-from text_generation_server.models.vlm_causal_lm import VlmCausalLM
-
-
-class Idefics2(VlmCausalLM):
-    def __init__(
-        self,
-        model_id: str,
-        revision: Optional[str] = None,
-        quantize: Optional[str] = None,
-        speculator: Optional[str] = None,
-        dtype: Optional[torch.dtype] = None,
-        trust_remote_code: bool = False,
-    ):
-        self.processor = AutoProcessor.from_pretrained(
-            model_id,
-            revision=revision,
-            trust_remote_code=trust_remote_code,
-            # XXX: Extremely important to cap resolution in order to limit
-            # VRAM usage.
-            size={"longest_edge": 448, "shortest_edge": 378},
-        )
-        super().__init__(
-            model_cls=Idefics2ForConditionalGeneration,
-            model_id=model_id,
-            revision=revision,
-            quantize=quantize,
-            speculator=speculator,
-            dtype=dtype,
-            trust_remote_code=trust_remote_code,
-        )
-
-    def get_layer_config(self, model) -> Tuple[int, int, int]:
-        return (
-            len(model.text_model.model.layers),
-            model.text_model.model.num_key_value_heads,
-            model.text_model.model.head_size,
-        )
-
-    def max_past(self) -> Optional[int]:
-        return getattr(self.model.text_model, "max_past", None)
diff --git a/server/text_generation_server/models/idefics_causal_lm.py b/server/text_generation_server/models/idefics_causal_lm.py
index e78a9655..9a7a6fe1 100644
--- a/server/text_generation_server/models/idefics_causal_lm.py
+++ b/server/text_generation_server/models/idefics_causal_lm.py
@@ -1,10 +1,12 @@
-import torch
+from io import BytesIO
+from PIL import Image
 import torch
 import time
 
 from dataclasses import dataclass
 from opentelemetry import trace
 from transformers import (
+    AutoConfig,
     AutoProcessor,
     AutoTokenizer,
     PreTrainedTokenizerBase,
@@ -21,11 +23,18 @@ from text_generation_server.models.types import (
 )
 from text_generation_server.pb import generate_pb2
 from text_generation_server.utils import NextTokenChooser, StoppingCriteria, Sampling
-from text_generation_server.models.vlm_causal_lm import split
+import torch.distributed
+from text_generation_server.models.custom_modeling.idefics_modeling import (
+    IdeficsForVisionText2Text,
+)
+from text_generation_server.utils import (
+    initialize_torch_distributed,
+    weight_files,
+    Weights,
+)
+from text_generation_server.utils.quantization import get_loader
 
-import re
-
-IMAGES = re.compile(r"!\[[^\]]*\]\((.*?)\s*(\"(?:.*[^\"])\")?\s*\)")
+from text_generation_server.utils.import_utils import SYSTEM
 
 
 tracer = trace.get_tracer(__name__)
@@ -109,7 +118,7 @@ class IdeficsCausalLMBatch(Batch):
         max_decode_tokens = 0
         for i, r in enumerate(pb.requests):
             requests_idx_mapping[r.id] = i
-            inputs.append(r.inputs)
+            inputs.append(r.input_chunks.chunks)
             next_token_choosers.append(
                 NextTokenChooser.from_pb(r.parameters, device, tokenizer)
             )
@@ -128,8 +137,15 @@ class IdeficsCausalLMBatch(Batch):
         for inp in inputs:
             # Each input is encoded into a list, where each element of this input list is either a string or a URL
             prompt = []
-            for chunk in split(inp):
-                prompt.append(chunk["content"])
+            for chunk in inp:
+                chunk_type = chunk.WhichOneof("chunk")
+                if chunk_type == "text":
+                    prompt.append(chunk.text)
+                elif chunk_type == "image":
+                    image = Image.open(BytesIO(chunk.image.data))
+                    prompt.append(image)
+                else:
+                    raise RuntimeError(f"Invalid chunk type {chunk_type}")
             prompts.append(prompt)
 
         # The processor replaces the call to tokenizer, and
@@ -286,7 +302,7 @@ class IdeficsCausalLMBatch(Batch):
             image_hidden_states = self.image_hidden_states[keep_indices]
 
         # Ensure that past_key_values tensors can be updated in-place
-        if type(self.past_key_values[0]) == tuple:
+        if type(self.past_key_values[0]) is tuple:
             self.past_key_values = [list(layer) for layer in self.past_key_values]
 
         # Update tensors in-place to allow incremental garbage collection
@@ -453,7 +469,7 @@ class IdeficsCausalLMBatch(Batch):
             # BLOOM Keys:   [batch_size * num_heads, head_dim, seq_length]
             # BLOOM Values: [batch_size * num_heads, seq_length, head_dim]
             # And ensure that we can update tensors in-place
-            if type(batch.past_key_values[0]) == tuple:
+            if isinstance(batch.past_key_values[0], tuple):
                 batch.past_key_values = [
                     [t.view(len(batch), -1, *t.shape[-2:]) for t in layer]
                     for layer in batch.past_key_values
@@ -574,22 +590,38 @@ class IdeficsCausalLM(Model):
         model_id: str,
         revision: Optional[str] = None,
         quantize: Optional[str] = None,
+        speculator: Optional[str] = None,
         dtype: Optional[torch.dtype] = None,
         trust_remote_code: bool = False,
     ):
-        from text_generation_server.models.custom_modeling.idefics_modeling import (
-            IdeficsForVisionText2Text,
-        )
-
+        self.quantize = quantize
+        self.process_group, rank, world_size = initialize_torch_distributed()
         if torch.cuda.is_available():
-            device = torch.device("cuda")
-            dtype = torch.bfloat16 if dtype is None else dtype
+            device = torch.device(f"cuda:{rank}")
+            # 9b seems to work correctly enough in float16, but 80b seems
+            # to be really saturating for f16.
+            dtype = torch.float16 if dtype is None else dtype
+        elif SYSTEM == "ipex":
+            if hasattr(torch, "xpu") and torch.xpu.is_available():
+                device = torch.device(f"xpu:{rank}")
+                dtype = torch.float16 if dtype is None else dtype
+            else:
+                device = torch.device("cpu")
+                # Float16 doesn't exist on target.
+                dtype = torch.bfloat16 if dtype is None else dtype
         else:
-            if quantize:
-                raise ValueError("quantization is not available on CPU")
-
             device = torch.device("cpu")
             dtype = torch.float32 if dtype is None else dtype
+        self.device, self.dtype = device, dtype
+
+        config = AutoConfig.from_pretrained(
+            model_id,
+            revision=revision,
+            trust_remote_code=trust_remote_code,
+        )
+        config.quantize = quantize
+        config.speculator = speculator
+        config.vision_config.quantize = quantize
 
         tokenizer = AutoTokenizer.from_pretrained(
             model_id,
@@ -605,37 +637,34 @@ class IdeficsCausalLM(Model):
             truncation_side="left",
             trust_remote_code=trust_remote_code,
         )
-        model = IdeficsForVisionText2Text.from_pretrained(
-            model_id,
-            revision=revision,
-            torch_dtype=dtype,
-            device_map=(
-                "auto"
-                if torch.cuda.is_available() and torch.cuda.device_count() > 1
-                else None
-            ),
-            load_in_8bit=quantize == "bitsandbytes",
-            trust_remote_code=trust_remote_code,
+
+        weights_loader = get_loader(
+            quantize=quantize, model_id=model_id, revision=revision
+        )
+        torch.distributed.barrier(group=self.process_group)
+        filenames = weight_files(model_id, revision=revision, extension=".safetensors")
+        weights = Weights(
+            filenames,
+            device=device,
+            dtype=dtype,
+            process_group=self.process_group,
+            weights_loader=weights_loader,
         )
-        if torch.cuda.is_available() and torch.cuda.device_count() == 1:
-            model = model.cuda()
 
-        if tokenizer.pad_token_id is None:
-            if model.config.pad_token_id is not None:
-                tokenizer.pad_token_id = model.config.pad_token_id
-            elif model.config.eos_token_id is not None:
-                tokenizer.pad_token_id = model.config.eos_token_id
-            elif tokenizer.eos_token_id is not None:
-                tokenizer.pad_token_id = tokenizer.eos_token_id
-            else:
-                tokenizer.add_special_tokens({"pad_token": "<unk>"})
+        model = IdeficsForVisionText2Text(config, weights)
 
-        super(IdeficsCausalLM, self).__init__(
+        self.config = config
+
+        torch.distributed.barrier(group=self.process_group)
+        super().__init__(
+            model_id=model_id,
             model=model,
             tokenizer=tokenizer,
             requires_padding=True,
             dtype=dtype,
             device=device,
+            rank=rank,
+            world_size=world_size,
         )
 
     @property
diff --git a/server/text_generation_server/models/llava_next.py b/server/text_generation_server/models/llava_next.py
deleted file mode 100644
index effe8b91..00000000
--- a/server/text_generation_server/models/llava_next.py
+++ /dev/null
@@ -1,46 +0,0 @@
-import torch
-
-from typing import Optional, Tuple
-
-from transformers import (
-    AutoProcessor,
-)
-from text_generation_server.models.custom_modeling.llava_next import (
-    LlavaNextForConditionalGeneration,
-)
-
-from text_generation_server.models.vlm_causal_lm import VlmCausalLM
-
-
-class LlavaNext(VlmCausalLM):
-    def __init__(
-        self,
-        model_id: str,
-        revision: Optional[str] = None,
-        quantize: Optional[str] = None,
-        speculator: Optional[str] = None,
-        dtype: Optional[torch.dtype] = None,
-        trust_remote_code: bool = False,
-    ):
-        self.processor = AutoProcessor.from_pretrained(
-            model_id, revision=revision, trust_remote_code=trust_remote_code
-        )
-        super().__init__(
-            model_cls=LlavaNextForConditionalGeneration,
-            model_id=model_id,
-            revision=revision,
-            quantize=quantize,
-            speculator=speculator,
-            dtype=dtype,
-            trust_remote_code=trust_remote_code,
-        )
-
-    def get_layer_config(self, model) -> Tuple[int, int, int]:
-        return (
-            len(model.language_model.model.layers),
-            model.language_model.model.num_key_value_heads,
-            model.language_model.model.head_size,
-        )
-
-    def max_past(self) -> Optional[int]:
-        return getattr(self.model.language_model, "max_past", None)
diff --git a/server/text_generation_server/models/mamba.py b/server/text_generation_server/models/mamba.py
index d9f90590..f6dcde68 100644
--- a/server/text_generation_server/models/mamba.py
+++ b/server/text_generation_server/models/mamba.py
@@ -2,7 +2,6 @@ import torch
 import torch.distributed
 from transformers import AutoTokenizer, PreTrainedTokenizerBase
 from typing import Optional
-import os
 from text_generation_server.models.custom_modeling.mamba_modeling import (
     MambaConfig,
 )
@@ -20,16 +19,18 @@ from text_generation_server.models.custom_modeling.mamba_modeling import (
     InferenceParams,
 )
 from text_generation_server.models import Model
-from typing import Any, List, Optional, Tuple, Type, Dict
+from typing import Any, List, Tuple, Type, Dict
 from text_generation_server.models.types import (
     Batch,
     Tokens,
     Generation,
     GeneratedText,
 )
+from text_generation_server.utils.chunks import concat_text_chunks
+from text_generation_server.utils.quantization import get_loader
 from text_generation_server.utils.tokens import batch_top_tokens, Sampling
 from dataclasses import dataclass
-from text_generation_server.utils import NextTokenChooser, StoppingCriteria, Sampling
+from text_generation_server.utils import NextTokenChooser, StoppingCriteria
 
 
 def new_inference_params(
@@ -139,7 +140,7 @@ class MambaBatch(Batch):
         max_decode_tokens = 0
         for i, r in enumerate(pb.requests):
             requests_idx_mapping[r.id] = i
-            inputs.append(r.inputs)
+            inputs.append(concat_text_chunks(r.input_chunks.chunks))
             next_token_choosers.append(
                 NextTokenChooser.from_pb(r.parameters, device, tokenizer)
             )
@@ -297,7 +298,6 @@ class MambaBatch(Batch):
         stopping_criterias = []
         top_n_tokens = []
         max_tokens = 0
-        max_seqlen = 0
         seqlen_offset = 0
 
         (n_blocks, _, d_inner, d_conv) = batches[0].inference_params.conv_states.shape
@@ -412,6 +412,7 @@ class Mamba(Model):
         dtype: Optional[torch.dtype] = None,
         trust_remote_code: bool = False,
     ):
+        self.quantize = quantize
         self.process_group, _rank, world_size = initialize_torch_distributed()
         if world_size > 1:
             raise RuntimeError("Mamba does not support Tensor Parallelism (TP)")
@@ -447,11 +448,21 @@ class Mamba(Model):
         config.quantize = quantize
         config.speculator = speculator
         torch.distributed.barrier(group=self.process_group)
+        weights_loader = get_loader(
+            quantize=quantize, model_id=model_id, revision=revision
+        )
         filenames = weight_files(model_id, revision=revision, extension=".safetensors")
-        weights = Weights(filenames, device, dtype, process_group=self.process_group)
+        weights = Weights(
+            filenames,
+            device,
+            dtype,
+            process_group=self.process_group,
+            weights_loader=weights_loader,
+        )
         model = MambaModel(config, weights)
         torch.distributed.barrier(group=self.process_group)
         super(Mamba, self).__init__(
+            model_id=model_id,
             model=model,
             tokenizer=tokenizer,
             requires_padding=True,
@@ -473,7 +484,7 @@ class Mamba(Model):
                     for bs in CUDA_GRAPHS:
                         self.cuda_graph_warmup(bs)
                 except Exception:
-                    logger.exception(f"Decode cuda graph warmup failed")
+                    logger.exception("Decode cuda graph warmup failed")
         else:
             logger.info(f"Cuda Graphs are disabled (CUDA_GRAPHS={CUDA_GRAPHS}).")
 
@@ -522,7 +533,7 @@ class Mamba(Model):
         }
         self.cuda_graphs[batch_size] = graph_dict
 
-    def tunableop_warmup(self, seqlen: int):
+    def tunableop_warmup(self, batch_size: int, seqlen: int):
         input_ids = torch.zeros((batch_size, 1), dtype=torch.int64, device=self.device)
         n_blocks = len(self.model.blocks)
 
diff --git a/server/text_generation_server/models/mllama_causal_lm.py b/server/text_generation_server/models/mllama_causal_lm.py
new file mode 100644
index 00000000..9e19e171
--- /dev/null
+++ b/server/text_generation_server/models/mllama_causal_lm.py
@@ -0,0 +1,357 @@
+from io import BytesIO
+from PIL import Image
+import torch
+from typing import Iterable, Optional, Tuple, List, Dict
+from text_generation_server.pb.generate_pb2 import Request
+
+from dataclasses import dataclass
+from opentelemetry import trace
+from transformers import (
+    PreTrainedTokenizerBase,
+)
+from text_generation_server.models.vlm_causal_lm import VlmCausalLMBatch, VlmCausalLM
+from text_generation_server.pb import generate_pb2
+from text_generation_server.models.flash_causal_lm import (
+    block_tables_to_ragged,
+)
+from text_generation_server.models.globals import PREFIX_CACHING, ATTENTION
+from text_generation_server.layers.attention import Seqlen
+
+
+tracer = trace.get_tracer(__name__)
+
+
+@dataclass
+class MllamaCausalLMBatch(VlmCausalLMBatch):
+    image_indices: List[int] = 42
+    aspect_ratio_ids: Optional[torch.Tensor] = None
+    aspect_ratio_mask: Optional[torch.Tensor] = None
+    cross_attention_states: Optional[torch.Tensor] = None
+
+    @classmethod
+    @tracer.start_as_current_span("concatenate")
+    def concatenate(cls, batches):
+        batch = super().concatenate(batches)
+        batch.pixel_values = None
+        batch.pixel_attention_mask = None
+
+        offset = 0
+        image_indices = []
+        attention_states = []
+        for b in batches:
+            if b.cross_attention_states is not None:
+                attention_states.append(b.cross_attention_states)
+            image_indices.extend([i + offset for i in b.image_indices])
+            offset += len(b.image_indices)
+        if len(attention_states) > 0:
+            assert len(image_indices) > 0
+            batch.cross_attention_states = torch.cat(attention_states, dim=0)
+            batch.image_indices = image_indices
+        else:
+            batch.cross_attention_states = None
+            batch.image_indices = []
+        return batch
+
+    @tracer.start_as_current_span("filter")
+    def filter(self, request_ids: List[int]):
+        assert self.image_indices is not None
+        batch = super().filter(request_ids)
+        assert self.image_indices is not None
+        indices = []
+        for i, request_id in enumerate(request_ids):
+            idx = self.requests_idx_mapping[request_id]
+            indices.append(idx)
+
+        offset = 0
+        new_image_indices = []
+        prev_i = None
+        for i in self.image_indices:
+            if i in indices:
+                new_image_indices.append(offset)
+                if i != prev_i:
+                    offset += 1
+                prev_i = i
+
+        batch.image_indices = new_image_indices
+        if len(new_image_indices) > 0:
+            assert max(new_image_indices) < self.cross_attention_states.shape[0]
+            assert offset <= self.cross_attention_states.shape[0]
+            batch.cross_attention_states = self.cross_attention_states[
+                new_image_indices
+            ]
+        else:
+            batch.cross_attention_states = None
+        return batch
+
+    @classmethod
+    def batch_tokenized_inputs(
+        cls, requests: Iterable[Request], tokenizer, processor, config
+    ):
+        image_inputs = []
+        texts = []
+        image_indices = []
+        batch_tokenized_inputs = []
+
+        for i, r in enumerate(requests):
+            # Each input is encoded into a list, where each element of this input list is either a string or a URL
+            curr_text = ""
+            curr_image = None
+            curr_i = None
+            for chunk in r.input_chunks.chunks:
+                chunk_type = chunk.WhichOneof("chunk")
+                if chunk_type == "text":
+                    curr_text += chunk.text
+                elif chunk_type == "image":
+                    image = Image.open(BytesIO(chunk.image.data))
+                    # TODO unsure about BOS
+                    curr_text += "<|image|>"
+                    image_input = processor.image_processor(image, return_tensors="pt")
+                    curr_image = image_input
+                    curr_i = i
+                    # image_inputs.append(image_input)
+                    # image_indices.append(i)
+                else:
+                    raise RuntimeError(f"Invalid chunk type {chunk_type}")
+            texts.append(curr_text)
+            if curr_image is not None:
+                image_inputs.append(curr_image)
+                image_indices.append(curr_i)
+
+            input_ids = tokenizer(
+                curr_text,
+                truncation=True,
+                max_length=r.truncate,
+                add_special_tokens=r.add_special_tokens,
+            )["input_ids"]
+            batch_tokenized_inputs.append(input_ids)
+        if image_inputs:
+            image_input = image_inputs[0]
+            new_image_inputs = {
+                "pixel_values": torch.cat(
+                    [img["pixel_values"] for img in image_inputs], dim=0
+                ),
+            }
+            if "aspect_ratio_ids" in image_input:
+                new_image_inputs["aspect_ratio_ids"] = torch.cat(
+                    [img["aspect_ratio_ids"] for img in image_inputs], dim=0
+                )
+            if "aspect_ratio_mask" in image_input:
+                new_image_inputs["aspect_ratio_mask"] = torch.cat(
+                    [img["aspect_ratio_mask"] for img in image_inputs], dim=0
+                )
+            image_inputs = new_image_inputs
+            image_inputs["image_indices"] = image_indices
+        else:
+            image_inputs = None
+
+        if image_inputs is not None:
+            assert len(image_indices) == image_inputs["pixel_values"].shape[0]
+
+        return batch_tokenized_inputs, image_inputs
+
+    @classmethod
+    def from_pb_processor(
+        cls,
+        pb: generate_pb2.Batch,
+        tokenizer: PreTrainedTokenizerBase,
+        processor,
+        config,
+        dtype: torch.dtype,
+        device: torch.device,
+    ) -> "VlmCausalLMBatch":
+        batch_tokenized_inputs, image_inputs = cls.batch_tokenized_inputs(
+            pb.requests, tokenizer, processor, config
+        )
+        batch = cls.from_tokenized(pb, tokenizer, batch_tokenized_inputs, dtype, device)
+        # XXX: <|image|> token is actually out of bounds and bugs out the logit processors.
+        batch.all_input_ids_tensor = batch.all_input_ids_tensor.clamp(
+            max=config.text_config.vocab_size - 1
+        )
+        batch.input_ids = batch.input_ids.clamp(max=config.text_config.vocab_size - 1)
+
+        if image_inputs is not None:
+            batch.pixel_values = image_inputs["pixel_values"].to(
+                device=device, dtype=dtype
+            )
+            batch.aspect_ratio_ids = image_inputs["aspect_ratio_ids"].to(device=device)
+            batch.aspect_ratio_mask = image_inputs["aspect_ratio_mask"].to(
+                device=device
+            )
+            batch.image_indices = image_inputs["image_indices"]
+        else:
+            batch.pixel_values = None
+            batch.aspect_ratio_ids = None
+            batch.aspect_ratio_mask = None
+            batch.image_indices = []
+        assert batch.image_indices is not None
+        return batch
+
+
+class MllamaCausalLM(VlmCausalLM):
+    def forward(
+        self,
+        batch: VlmCausalLMBatch,
+        adapter_data: Optional[Dict[str, torch.Tensor]] = None,
+    ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
+        # Model Forward
+        if batch.speculative_ids is not None:
+            input_ids = batch.input_ids
+            position_ids = batch.position_ids
+            cu_seqlen_prefill = batch.cu_seqlen_prefill
+            kv_cache = self.kv_cache
+            block_tables = batch.block_tables_tensor
+            slots = batch.slots[batch.slot_indices]
+            input_lengths = batch.input_lengths_tensor
+            max_s = batch.max_seqlen
+            lm_head_indices = batch.prefill_head_indices
+
+            speculative_ids = batch.speculative_ids
+
+            B, speculative_length = speculative_ids.shape
+            new_length = speculative_length + 1
+            new_input_ids = torch.cat(
+                [input_ids.unsqueeze(-1), speculative_ids], dim=1
+            ).reshape(-1)
+            arange = torch.arange(new_length, device=position_ids.device).unsqueeze(0)
+            arange_int = arange.to(dtype=torch.int32)
+            new_position_ids = (
+                position_ids.unsqueeze(-1).expand(B, new_length) + arange
+            ).view(-1)
+            slots = (slots.unsqueeze(-1).expand(B, new_length) + arange_int).view(-1)
+            input_lengths = (
+                input_lengths.unsqueeze(-1).expand(B, new_length) + arange_int
+            ).view(-1)
+            prefix_lens_tensor = (
+                batch.prefix_lens_tensor.unsqueeze(-1).expand(B, new_length)
+            ).reshape(-1)
+
+            # Add Copy the block tables for all members
+            block_tables = (
+                block_tables.unsqueeze(1)
+                .expand(B, new_length, -1)
+                .reshape(B * new_length, -1)
+                .contiguous()
+            )
+            max_s = max_s + speculative_length
+
+            input_ids = new_input_ids
+            position_ids = new_position_ids
+        else:
+            input_ids = batch.input_ids
+            position_ids = batch.position_ids
+            cu_seqlen_prefill = batch.cu_seqlen_prefill
+            kv_cache = self.kv_cache
+            block_tables = batch.block_tables_tensor
+            slots = batch.slots[batch.slot_indices]
+            input_lengths = batch.input_lengths_tensor
+            prefix_lens_tensor = batch.prefix_lens_tensor
+            max_s = batch.max_seqlen
+            lm_head_indices = batch.prefill_head_indices
+
+        if cu_seqlen_prefill is None and self.max_past() is not None:
+            # In decode, not prefill, we're actually overwriting the KV-cache
+            # in a circular buffer mode.
+            # This makes sure the max_s for the decode pass is correct.
+            max_s = min(self.max_past(), max_s)
+
+        bs = input_ids.shape[0]
+        # Try to find an associated cuda graph
+        bs = input_ids.shape[0]
+        sorted_padded_bs = sorted([k for k in self.cuda_graphs.keys() if k >= bs])
+        if sorted_padded_bs:
+            # Get associated cuda graph
+            cuda_graph = self.cuda_graphs[sorted_padded_bs[0]]
+        else:
+            cuda_graph = None
+        if (
+            cu_seqlen_prefill is not None
+            or cuda_graph is None
+            # Only run cuda graphs when there's no images.
+            or batch.cross_attention_states is not None
+        ):
+            input_lengths = input_lengths + prefix_lens_tensor
+            if PREFIX_CACHING:
+                block_tables = block_tables_to_ragged(
+                    block_tables=block_tables,
+                    input_lengths=batch.input_lengths,
+                    prefix_lens=batch.prefix_lens,
+                )
+            with self._forward_context(
+                block_tables=block_tables,
+                cu_seqlen_prefill=cu_seqlen_prefill,
+                input_lengths_tensor=input_lengths,
+                prefix_lens_tensor=prefix_lens_tensor,
+            ):
+                max_k = (input_lengths + prefix_lens_tensor).max().item()
+                seqlen = Seqlen(
+                    input_lengths=input_lengths,
+                    prefix_lengths=prefix_lens_tensor,
+                    cu_seqlen_q=cu_seqlen_prefill,
+                    max_q=max_s,
+                    max_k=max_k,
+                )
+
+                if batch.pixel_values is not None:
+                    cross_attention_states = self.model.vision_forward(
+                        pixel_values=batch.pixel_values,
+                        aspect_ratio_ids=batch.aspect_ratio_ids,
+                        aspect_ratio_mask=batch.aspect_ratio_mask,
+                    )
+                    batch.cross_attention_states = cross_attention_states
+
+                cross_attention_states = batch.cross_attention_states
+
+                logits, speculative_logits = self.model.forward(
+                    input_ids=input_ids,
+                    position_ids=position_ids,
+                    cu_seqlen_prefill=cu_seqlen_prefill,
+                    kv_cache=kv_cache,
+                    block_tables=block_tables,
+                    slots=slots,
+                    seqlen=seqlen,
+                    max_s=max_s,
+                    prefill_cache_indices=batch.prefill_cache_indices,
+                    lm_head_indices=lm_head_indices,
+                    cross_attention_states=cross_attention_states,
+                    adapter_data=adapter_data,
+                    image_indices=batch.image_indices[:],
+                )
+                if batch.prefill_cache_indices is not None:
+                    batch.prefill_cache_indices = None
+                if batch.pixel_values is not None:
+                    batch.pixel_values = None
+                return logits, speculative_logits
+
+        # Copy inputs to the static inputs of the cuda graph
+        # Static inputs are potentially padded
+        cuda_graph["input_ids"][: input_ids.shape[0]] = input_ids
+        cuda_graph["position_ids"][: position_ids.shape[0]] = position_ids
+        if ATTENTION == "flashinfer":
+            block_tables = block_tables_to_ragged(
+                block_tables=block_tables,
+                input_lengths=batch.input_lengths,
+                prefix_lens=batch.prefix_lens,
+            )
+            cuda_graph["block_tables"][: block_tables.shape[0]] = block_tables
+        else:
+            cuda_graph["block_tables"][
+                : block_tables.shape[0], : block_tables.shape[1]
+            ] = block_tables
+        cuda_graph["slots"].fill_(0)
+        cuda_graph["slots"][: slots.shape[0]] = slots
+        cuda_graph["input_lengths"].zero_()
+        cuda_graph["input_lengths"][: input_lengths.shape[0]] = (
+            input_lengths + prefix_lens_tensor
+        )
+
+        # Replay the graph
+        cuda_graph["graph"].replay()
+
+        # Slice output to the correct shape
+        speculative_logits = (
+            cuda_graph["speculative_logits"][:bs]
+            if cuda_graph["speculative_logits"] is not None
+            else None
+        )
+        logits = cuda_graph["logits"][:bs]
+        return logits, speculative_logits
diff --git a/server/text_generation_server/models/model.py b/server/text_generation_server/models/model.py
index 7546e70b..4568b8dd 100644
--- a/server/text_generation_server/models/model.py
+++ b/server/text_generation_server/models/model.py
@@ -1,13 +1,19 @@
 import inspect
+from loguru import logger
 import torch
 
 from abc import ABC, abstractmethod
-from typing import List, Optional, Tuple, Type, TypeVar
+from typing import List, Tuple, Optional, TypeVar, Type, Dict
+from collections import defaultdict
 from transformers import PreTrainedTokenizerBase
 
 from text_generation_server.models.types import Batch, Generation
 from text_generation_server.utils.speculate import get_speculate
 from text_generation_server.pb.generate_pb2 import InfoResponse
+from text_generation_server.adapters.weights import LayerAdapterWeights
+import time
+BASE_MODEL_ADAPTER_ID = "__base_model__"
+
 
 B = TypeVar("B", bound=Batch)
 
@@ -15,6 +21,7 @@ B = TypeVar("B", bound=Batch)
 class Model(ABC):
     def __init__(
         self,
+        model_id: str,
         model: torch.nn.Module,
         tokenizer: PreTrainedTokenizerBase,
         requires_padding: bool,
@@ -22,10 +29,12 @@ class Model(ABC):
         device: torch.device,
         rank: int = 0,
         world_size: int = 1,
-        kwargs: dict = {},
+        sliding_window: Optional[int] = None,
         speculate: Optional[int] = None,
+        adapter_id: str = BASE_MODEL_ADAPTER_ID,
     ):
-        self.model = model
+        self.model_id = model_id
+        self.model = model.eval()
         self.tokenizer = tokenizer
 
         # all_special_ids is not set correctly if the rust tokenizer is unpacked
@@ -40,7 +49,14 @@ class Model(ABC):
         self.device = device
         self.rank = rank
         self.world_size = world_size
-        self.kwargs = kwargs
+        self.sliding_window = sliding_window if sliding_window != -1 else None
+
+        self.layer_to_adapter_weights: Dict[str, LayerAdapterWeights] = defaultdict(
+            LayerAdapterWeights
+        )
+        self.loaded_adapters = set()
+        self.static_adapter_id = adapter_id
+
         if speculate is None:
             speculate = get_speculate()
         self.speculate = speculate
@@ -54,10 +70,14 @@ class Model(ABC):
 
     @property
     def info(self) -> InfoResponse:
+        if self.requires_padding and self.sliding_window is not None:
+            raise NotImplementedError("sliding_window is not implemented with padding")
+
         return InfoResponse(
             requires_padding=self.requires_padding,
             dtype=str(self.dtype),
             device_type=self.device.type,
+            window_size=self.sliding_window,
             speculate=self.speculate,
         )
 
@@ -72,21 +92,29 @@ class Model(ABC):
     ) -> Tuple[List[Generation], Optional[B], Tuple[int, int]]:
         raise NotImplementedError
 
-    def warmup(self, batch: B, max_total_tokens: int):
+    def warmup(self, batch: B) -> Optional[int]:
         self.generate_token(batch)
+        return None
 
     def decode_token(
         self,
         all_input_ids: List[int],
         prefix_offset: int = 0,
         read_offset: int = 0,
+        skip_special_tokens: bool = False,
     ) -> Tuple[str, int, int]:
         """Hack to hopefully support generate_stream for the maximum number of tokenizers"""
 
         # The prefix text is necessary only to defeat cleanup algorithms in the decode
         # which decide to add a space or not depending on the surrounding ids.
-        prefix_text = self.tokenizer.decode(all_input_ids[prefix_offset:read_offset], skip_special_tokens=False)
-        new_text = self.tokenizer.decode(all_input_ids[prefix_offset:], skip_special_tokens=False)
+        prefix_text = self.tokenizer.decode(
+            all_input_ids[prefix_offset:read_offset],
+            skip_special_tokens=skip_special_tokens,
+        )
+
+        new_text = self.tokenizer.decode(
+            all_input_ids[prefix_offset:], skip_special_tokens=skip_special_tokens
+        )
 
         if len(new_text) > len(prefix_text) and not new_text.endswith("�"):
             # utf-8 char at the end means it's a potential unfinished byte sequence
diff --git a/server/text_generation_server/models/mpt.py b/server/text_generation_server/models/mpt.py
deleted file mode 100644
index 8d8b4909..00000000
--- a/server/text_generation_server/models/mpt.py
+++ /dev/null
@@ -1,104 +0,0 @@
-import torch
-import torch.distributed
-
-from pathlib import Path
-from typing import Optional, Type
-from opentelemetry import trace
-from transformers import AutoTokenizer, PretrainedConfig, PreTrainedTokenizerBase
-from huggingface_hub import hf_hub_download
-import json
-
-from text_generation_server.models import CausalLM
-from text_generation_server.models.causal_lm import CausalLMBatch
-from text_generation_server.pb import generate_pb2
-from text_generation_server.models.custom_modeling.mpt_modeling import (
-    MPTForCausalLM,
-)
-from text_generation_server.utils import (
-    initialize_torch_distributed,
-    weight_files,
-    Weights,
-)
-
-tracer = trace.get_tracer(__name__)
-
-
-class MPTCausalLMBatch(CausalLMBatch):
-    @classmethod
-    def from_pb(
-        cls,
-        pb: generate_pb2.Batch,
-        tokenizer: PreTrainedTokenizerBase,
-        dtype: torch.dtype,
-        device: torch.device,
-    ) -> "CausalLMBatch":
-        batch = super().from_pb(pb=pb, tokenizer=tokenizer, dtype=dtype, device=device)
-        batch.keys_head_dim_last = False
-        return batch
-
-
-class MPTSharded(CausalLM):
-    def __init__(
-        self,
-        model_id: str,
-        revision: Optional[str] = None,
-        quantize: Optional[str] = None,
-        speculator: Optional[str] = None,
-        dtype: Optional[torch.dtype] = None,
-        trust_remote_code: bool = False,
-    ):
-        self.process_group, rank, world_size = initialize_torch_distributed()
-        if torch.cuda.is_available():
-            device = torch.device(f"cuda:{rank}")
-            dtype = torch.float16 if dtype is None else dtype
-        else:
-            device = torch.device("cpu")
-            dtype = torch.float32 if dtype is None else dtype
-
-        tokenizer = AutoTokenizer.from_pretrained(
-            model_id,
-            revision=revision,
-            padding_side="left",
-            truncation_side="left",
-            trust_remote_code=trust_remote_code,
-        )
-        tokenizer.pad_token = tokenizer.eos_token
-
-        # If model_id is a local path, load the file directly
-        local_path = Path(model_id, "config.json")
-        if local_path.exists():
-            filename = str(local_path.resolve())
-        else:
-            filename = hf_hub_download(
-                model_id, revision=revision, filename="config.json"
-            )
-        with open(filename, "r") as f:
-            config = json.load(f)
-        config = PretrainedConfig(**config)
-        config.quantize = quantize
-        config.speculator = speculator
-
-        torch.distributed.barrier(group=self.process_group)
-
-        filenames = weight_files(model_id, revision=revision, extension=".safetensors")
-        weights = Weights(filenames, device, dtype, process_group=self.process_group)
-        if config.quantize == "gptq":
-            weights._set_gptq_params(model_id, revision)
-
-        config.quantize = quantize
-        model = MPTForCausalLM(config, weights)
-
-        torch.distributed.barrier(group=self.process_group)
-        super(CausalLM, self).__init__(
-            model=model,
-            tokenizer=tokenizer,
-            requires_padding=False,
-            dtype=dtype,
-            device=device,
-            rank=rank,
-            world_size=world_size,
-        )
-
-    @property
-    def batch_type(self) -> Type[CausalLMBatch]:
-        return MPTCausalLMBatch
diff --git a/server/text_generation_server/models/opt.py b/server/text_generation_server/models/opt.py
deleted file mode 100644
index 5b84f4ff..00000000
--- a/server/text_generation_server/models/opt.py
+++ /dev/null
@@ -1,85 +0,0 @@
-import torch
-import torch.distributed
-
-from typing import Optional
-
-from transformers import (
-    AutoTokenizer,
-    AutoConfig,
-)
-from text_generation_server.models.custom_modeling.opt_modeling import OPTForCausalLM
-from text_generation_server.models import CausalLM
-from text_generation_server.utils import (
-    initialize_torch_distributed,
-    weight_files,
-    Weights,
-)
-
-
-class OPTSharded(CausalLM):
-    def __init__(
-        self,
-        model_id: str,
-        revision: Optional[str] = None,
-        quantize: Optional[str] = None,
-        speculator: Optional[str] = None,
-        dtype: Optional[torch.dtype] = None,
-        trust_remote_code: bool = False,
-    ):
-        self.process_group, rank, world_size = initialize_torch_distributed()
-        if torch.cuda.is_available():
-            device = torch.device(f"cuda:{rank}")
-            dtype = torch.float16 if dtype is None else dtype
-        else:
-            device = torch.device("cpu")
-            dtype = torch.float32 if dtype is None else dtype
-
-        tokenizer = AutoTokenizer.from_pretrained(
-            model_id,
-            revision=revision,
-            padding_side="left",
-            truncation_side="left",
-            trust_remote_code=trust_remote_code,
-        )
-
-        config = AutoConfig.from_pretrained(
-            model_id,
-            revision=revision,
-            trust_remote_code=trust_remote_code,
-        )
-        config.quantize = quantize
-        config.speculator = speculator
-        tokenizer.pad_token_id = config.pad_token_id
-
-        torch.distributed.barrier(group=self.process_group)
-        filenames = weight_files(model_id, revision=revision, extension=".safetensors")
-        weights = Weights(
-            filenames, device=device, dtype=dtype, process_group=self.process_group
-        )
-        if config.quantize == "gptq":
-            weights._set_gptq_params(model_id, revision)
-
-        model = OPTForCausalLM(config, weights)
-
-        torch.distributed.barrier(group=self.process_group)
-        super(CausalLM, self).__init__(
-            model=model,
-            tokenizer=tokenizer,
-            requires_padding=True,
-            dtype=dtype,
-            device=device,
-            rank=rank,
-            world_size=world_size,
-        )
-
-    def forward(
-        self, input_ids, attention_mask, position_ids, past_key_values: Optional = None
-    ):
-        outputs = self.model.forward(
-            input_ids=input_ids,
-            attention_mask=attention_mask,
-            past_key_values=past_key_values,
-            use_cache=True,
-        )
-
-        return outputs.logits, outputs.past_key_values
diff --git a/server/text_generation_server/models/pali_gemma.py b/server/text_generation_server/models/pali_gemma.py
index d94b9526..fe75570e 100644
--- a/server/text_generation_server/models/pali_gemma.py
+++ b/server/text_generation_server/models/pali_gemma.py
@@ -1,55 +1,45 @@
+from io import BytesIO
+from PIL import Image
 import torch
 import torch.distributed
 from opentelemetry import trace
-from typing import Optional, Tuple
+from typing import Iterable
 from text_generation_server.models.vlm_causal_lm import (
-    VlmCausalLM,
     VlmCausalLMBatch,
     image_text_replacement,
-    load_data_uri,
-    split,
 )
-from text_generation_server.models.custom_modeling.flash_pali_gemma_modeling import (
-    PaliGemmaForConditionalGeneration,
-)
-from transformers import AutoProcessor, AutoConfig, AutoImageProcessor
+
+from text_generation_server.pb.generate_pb2 import Request
 
 tracer = trace.get_tracer(__name__)
 
 
 class PaliGemmaBatch(VlmCausalLMBatch):
     @classmethod
-    def batch_tokenized_inputs(cls, requests, tokenizer, processor, config):
+    def batch_tokenized_inputs(
+        cls, requests: Iterable[Request], tokenizer, processor, config
+    ):
         batch_inputs = []
         image_inputs = []
         max_truncation = 0
         for r in requests:
-            chunks = split(r.inputs)
             full_text = ""
             image_id = 0
-            for chunk in chunks:
-                if chunk["type"] == "text":
-                    full_text += "<bos>" + chunk["content"] + "\n"
-                elif chunk["type"] == "image":
-                    image = chunk["content"]
-                    # Should never receive URLs anymore, processing should be done
-                    # On the rust layer.
-                    # This avoid making n queries per TP
-                    # if image.startswith("https://") or image.startswith("http://"):
-                    #     image = processor.image_processor.fetch_images(image)
-                    if image.startswith("data:"):
-                        image = load_data_uri(image)
-                    else:
-                        raise RuntimeError(
-                            "Cannot process input image not starting with data:"
-                        )
+            for chunk in r.input_chunks.chunks:
+                chunk_type = chunk.WhichOneof("chunk")
+                if chunk_type == "text":
+                    full_text += "<bos>" + chunk.text + "\n"
+                elif chunk_type == "image":
+                    image = Image.open(BytesIO(chunk.image.data))
                     # TODO do_convert_RGB should be on by default ?
                     image = image.convert("RGB")
                     image_input = processor.image_processor(image, return_tensors="pt")
-                    full_text += image_text_replacement(image_input, config, image_id)
+                    full_text += image_text_replacement(
+                        processor, image_input, config, image_id
+                    )
                     image_inputs.append(image_input)
                 else:
-                    raise RuntimeError(f"Invalid chunk type {chunk['type']}")
+                    raise RuntimeError(f"Invalid chunk type {chunk_type}")
 
             batch_inputs.append(full_text)
             max_truncation = max(max_truncation, r.truncate)
@@ -79,45 +69,3 @@ class PaliGemmaBatch(VlmCausalLMBatch):
         else:
             image_inputs = None
         return batch_tokenized_inputs, image_inputs
-
-
-class PaliGemma(VlmCausalLM):
-    def __init__(
-        self,
-        model_id: str,
-        revision: Optional[str] = None,
-        quantize: Optional[str] = None,
-        speculator: Optional[str] = None,
-        dtype: Optional[torch.dtype] = None,
-        trust_remote_code: bool = False,
-    ):
-        self.processor = AutoProcessor.from_pretrained(
-            model_id,
-            revision=revision,
-            trust_remote_code=trust_remote_code,
-        )
-
-        super().__init__(
-            config_cls=AutoConfig,
-            model_cls=PaliGemmaForConditionalGeneration,
-            model_id=model_id,
-            revision=revision,
-            quantize=quantize,
-            speculator=speculator,
-            dtype=dtype,
-            trust_remote_code=trust_remote_code,
-        )
-
-    @property
-    def batch_type(self):
-        return PaliGemmaBatch
-
-    def get_layer_config(self, model) -> Tuple[int, int, int]:
-        return (
-            len(model.text_model.model.layers),
-            model.text_model.model.num_key_value_heads,
-            model.text_model.model.head_size,
-        )
-
-    def max_past(self) -> Optional[int]:
-        return getattr(self.model.text_model, "max_past", None)
diff --git a/server/text_generation_server/models/phi.py b/server/text_generation_server/models/phi.py
deleted file mode 100644
index d68866c1..00000000
--- a/server/text_generation_server/models/phi.py
+++ /dev/null
@@ -1,68 +0,0 @@
-import torch
-import torch.distributed
-
-from transformers import AutoConfig, AutoTokenizer
-from typing import Optional, List, Tuple
-
-from text_generation_server.models import CausalLM
-from text_generation_server.models.custom_modeling.phi_modeling import (
-    PhiConfig,
-    PhiForCausalLM,
-)
-from text_generation_server.utils import (
-    initialize_torch_distributed,
-    weight_files,
-    Weights,
-)
-
-
-class Phi(CausalLM):
-    def __init__(
-        self,
-        model_id: str,
-        revision: Optional[str] = None,
-        quantize: Optional[str] = None,
-        speculator: Optional[str] = None,
-        dtype: Optional[torch.dtype] = None,
-        trust_remote_code: bool = False,
-    ):
-        self.process_group, _rank, _world_size = initialize_torch_distributed()
-        if torch.cuda.is_available():
-            device = torch.device("cuda")
-            dtype = torch.float16 if dtype is None else dtype
-        else:
-            if quantize:
-                raise ValueError("quantization is not available on CPU")
-
-            device = torch.device("cpu")
-            dtype = torch.float32 if dtype is None else dtype
-
-        tokenizer = AutoTokenizer.from_pretrained(
-            model_id,
-            revision=revision,
-            padding_side="left",
-            truncation_side="left",
-            trust_remote_code=trust_remote_code,
-        )
-        config = PhiConfig.from_pretrained(
-            model_id, revision=revision, trust_remote_code=trust_remote_code
-        )
-
-        tokenizer.bos_token_id = config.bos_token_id
-        tokenizer.eos_token_id = config.eos_token_id
-        tokenizer.pad_token = tokenizer.eos_token
-
-        config.quantize = quantize
-        config.speculator = speculator
-        torch.distributed.barrier(group=self.process_group)
-        filenames = weight_files(model_id, revision=revision, extension=".safetensors")
-        weights = Weights(filenames, device, dtype, process_group=self.process_group)
-        model = PhiForCausalLM(config, weights)
-        torch.distributed.barrier(group=self.process_group)
-        super(CausalLM, self).__init__(
-            model=model,
-            tokenizer=tokenizer,
-            requires_padding=True,
-            dtype=dtype,
-            device=device,
-        )
diff --git a/server/text_generation_server/models/rw.py b/server/text_generation_server/models/rw.py
deleted file mode 100644
index d4764ded..00000000
--- a/server/text_generation_server/models/rw.py
+++ /dev/null
@@ -1,81 +0,0 @@
-import torch
-
-from transformers import AutoTokenizer, AutoModelForCausalLM
-from typing import List, Optional, Tuple
-
-from text_generation_server.models import CausalLM
-
-
-class RW(CausalLM):
-    def __init__(
-        self,
-        model_id: str,
-        revision: Optional[str] = None,
-        quantize: Optional[str] = None,
-        speculator: Optional[str] = None,
-        dtype: Optional[torch.dtype] = None,
-        trust_remote_code: bool = False,
-    ):
-        if speculator:
-            raise RuntimeError("Medusa decoding is not enabled for AutoModel")
-
-        if torch.cuda.is_available():
-            device = torch.device("cuda")
-            dtype = torch.float16 if dtype is None else dtype
-        else:
-            if quantize:
-                raise ValueError("quantization is not available on CPU")
-
-            device = torch.device("cpu")
-            dtype = torch.float32 if dtype is None else dtype
-
-        tokenizer = AutoTokenizer.from_pretrained(
-            model_id,
-            revision=revision,
-            padding_side="left",
-            truncation_side="left",
-            trust_remote_code=trust_remote_code,
-        )
-        model = AutoModelForCausalLM.from_pretrained(
-            model_id,
-            revision=revision,
-            torch_dtype=dtype,
-            device_map=(
-                "auto"
-                if torch.cuda.is_available() and torch.cuda.device_count() > 1
-                else None
-            ),
-            load_in_8bit=quantize == "bitsandbytes",
-            trust_remote_code=trust_remote_code,
-        )
-        if torch.cuda.is_available() and torch.cuda.device_count() == 1:
-            model = model.cuda()
-
-        if tokenizer.pad_token_id is None:
-            if model.config.pad_token_id is not None:
-                tokenizer.pad_token_id = model.config.pad_token_id
-            elif model.config.eos_token_id is not None:
-                tokenizer.pad_token_id = model.config.eos_token_id
-            elif tokenizer.eos_token_id is not None:
-                tokenizer.pad_token_id = tokenizer.eos_token_id
-            else:
-                tokenizer.add_special_tokens({"pad_token": "[PAD]"})
-
-        super(CausalLM, self).__init__(
-            model=model,
-            tokenizer=tokenizer,
-            requires_padding=True,
-            dtype=dtype,
-            device=device,
-        )
-
-    def forward(
-        self, input_ids, attention_mask, position_ids, past_key_values: Optional = None
-    ) -> Tuple[torch.Tensor, List[Tuple[torch.Tensor, torch.Tensor]]]:
-        # Model Forward
-        outputs = self.model.forward(
-            input_ids=input_ids,
-            attention_mask=attention_mask,
-            past_key_values=past_key_values,
-        )
-        return outputs.logits, outputs.past_key_values
diff --git a/server/text_generation_server/models/santacoder.py b/server/text_generation_server/models/santacoder.py
deleted file mode 100644
index f07734c2..00000000
--- a/server/text_generation_server/models/santacoder.py
+++ /dev/null
@@ -1,45 +0,0 @@
-from typing import Optional, List
-import torch
-
-from text_generation_server.models import CausalLM
-
-FIM_PREFIX = "<fim-prefix>"
-FIM_MIDDLE = "<fim-middle>"
-FIM_SUFFIX = "<fim-suffix>"
-FIM_PAD = "<fim-pad>"
-EOD = "<|endoftext|>"
-
-
-class SantaCoder(CausalLM):
-    def __init__(
-        self,
-        model_id: str,
-        revision: Optional[str] = None,
-        speculator: Optional[str] = None,
-        dtype: Optional[torch.dtype] = None,
-        trust_remote_code: bool = False,
-    ):
-        super().__init__(
-            model_id=model_id,
-            revision=revision,
-            speculator=speculator,
-            dtype=dtype,
-            trust_remote_code=trust_remote_code,
-        )
-
-        self.tokenizer.add_special_tokens(
-            {
-                "additional_special_tokens": [
-                    EOD,
-                    FIM_PREFIX,
-                    FIM_MIDDLE,
-                    FIM_SUFFIX,
-                    FIM_PAD,
-                ],
-                "pad_token": EOD,
-            }
-        )
-
-    def decode(self, generated_ids: List[int]) -> str:
-        # Do not skip special tokens as they are used for custom parsing rules of the generated text
-        return self.tokenizer.decode(generated_ids, skip_special_tokens=False, clean_up_tokenization_spaces=False)
diff --git a/server/text_generation_server/models/seq2seq_lm.py b/server/text_generation_server/models/seq2seq_lm.py
index 6a0c812f..04d4c28b 100644
--- a/server/text_generation_server/models/seq2seq_lm.py
+++ b/server/text_generation_server/models/seq2seq_lm.py
@@ -1,11 +1,23 @@
 import torch
+import torch.distributed
 import time
-
 from dataclasses import dataclass
 from opentelemetry import trace
-from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, PreTrainedTokenizerBase
+from transformers import (
+    AutoTokenizer,
+    AutoModelForSeq2SeqLM,
+    PreTrainedTokenizerBase,
+    AutoConfig,
+)
 from typing import Optional, Tuple, List, Type, Dict
-
+from text_generation_server.utils.import_utils import SYSTEM
+from text_generation_server.utils import (
+    initialize_torch_distributed,
+    weight_files,
+    Weights,
+)
+from text_generation_server.utils.chunks import concat_text_chunks
+from text_generation_server.utils.quantization import get_loader
 from text_generation_server.utils.tokens import batch_top_tokens
 from text_generation_server.models import Model
 from text_generation_server.models.types import (
@@ -93,7 +105,7 @@ class Seq2SeqLMBatch(Batch):
         padding_right_offset = 0
         max_decode_tokens = 0
         for i, r in enumerate(pb.requests):
-            inputs.append(r.inputs)
+            inputs.append(concat_text_chunks(r.input_chunks.chunks))
             requests_idx_mapping[r.id] = i
             decoder_input_lengths.append(1)
             next_token_choosers.append(
@@ -241,7 +253,7 @@ class Seq2SeqLMBatch(Batch):
         ]
 
         # Ensure that past_key_values tensors can be updated in-place
-        if type(self.past_key_values[0]) == tuple:
+        if type(self.past_key_values[0]) is tuple:
             self.past_key_values = [
                 [t for t in layer] for layer in self.past_key_values
             ]
@@ -417,7 +429,7 @@ class Seq2SeqLMBatch(Batch):
             batch.encoder_last_hidden_state = None
 
             # Ensure that we can update tensors in-place
-            if type(batch.past_key_values[0]) == tuple:
+            if isinstance(batch.past_key_values[0], tuple):
                 batch.past_key_values = [
                     [t for t in layer] for layer in batch.past_key_values
                 ]
@@ -530,6 +542,85 @@ class Seq2SeqLM(Model):
     def __init__(
         self,
         model_id: str,
+        model_class,
+        revision: Optional[str] = None,
+        quantize: Optional[str] = None,
+        speculator: Optional[str] = None,
+        dtype: Optional[torch.dtype] = None,
+        default_dtype=torch.float16,
+        trust_remote_code: bool = False,
+        config_class=AutoConfig,
+        tokenizer_class=AutoTokenizer,
+        aliases=None,
+    ):
+        self.quantize = quantize
+        self.process_group, rank, world_size = initialize_torch_distributed()
+        if torch.cuda.is_available():
+            device = torch.device(f"cuda:{rank}")
+            dtype = default_dtype if dtype is None else dtype
+        elif SYSTEM == "ipex":
+            if hasattr(torch, "xpu") and torch.xpu.is_available():
+                device = torch.device(f"xpu:{rank}")
+                dtype = default_dtype if dtype is None else dtype
+            else:
+                device = torch.device("cpu")
+                # Float16 doesn't exist on target.
+                dtype = torch.bfloat16 if dtype is None else dtype
+        else:
+            device = torch.device("cpu")
+            dtype = torch.float32 if dtype is None else dtype
+
+        config = config_class.from_pretrained(
+            model_id,
+            revision=revision,
+            trust_remote_code=trust_remote_code,
+        )
+        config.quantize = quantize
+        config.speculator = speculator
+
+        tokenizer = tokenizer_class.from_pretrained(
+            model_id,
+            revision=revision,
+            padding_side="left",
+            truncation_side="left",
+            trust_remote_code=trust_remote_code,
+        )
+        tokenizer.bos_token_id = config.decoder_start_token_id
+
+        weights_loader = get_loader(
+            quantize=quantize, model_id=model_id, revision=revision
+        )
+        torch.distributed.barrier(group=self.process_group)
+        filenames = weight_files(model_id, revision=revision, extension=".safetensors")
+        weights = Weights(
+            filenames,
+            device=device,
+            dtype=dtype,
+            process_group=self.process_group,
+            aliases=aliases,
+            weights_loader=weights_loader,
+        )
+        if config.quantize in ["awq", "exl2", "gptq", "marlin"]:
+            weights._set_gptq_params(model_id, revision)
+
+        model = model_class(config, weights)
+
+        torch.distributed.barrier(group=self.process_group)
+        super().__init__(
+            model_id=model_id,
+            model=model,
+            tokenizer=tokenizer,
+            requires_padding=True,
+            dtype=dtype,
+            device=device,
+            rank=rank,
+            world_size=world_size,
+        )
+
+    @classmethod
+    def fallback(
+        cls,
+        model_id: str,
         revision: Optional[str] = None,
         quantize: Optional[str] = None,
         speculator: Optional[str] = None,
@@ -573,23 +664,25 @@ class Seq2SeqLM(Model):
         )
         tokenizer.bos_token_id = model.config.decoder_start_token_id
 
-        super(Seq2SeqLM, self).__init__(
+        self = cls.__new__(
+            cls,
+        )
+        super().__init__(
+            self,
+            model_id=model_id,
             model=model,
             tokenizer=tokenizer,
             requires_padding=True,
             dtype=dtype,
             device=device,
         )
+        self.quantize = quantize
+        return self
 
     @property
     def batch_type(self) -> Type[Seq2SeqLMBatch]:
         return Seq2SeqLMBatch
 
-    def decode(self, decoder_ids: List[int]) -> str:
-        return self.tokenizer.decode(
-            decoder_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False
-        )
-
     def forward(
         self,
         input_ids,
diff --git a/server/text_generation_server/models/t5.py b/server/text_generation_server/models/t5.py
deleted file mode 100644
index 8e0735e5..00000000
--- a/server/text_generation_server/models/t5.py
+++ /dev/null
@@ -1,114 +0,0 @@
-import torch
-import torch.distributed
-
-from typing import List, Optional, Tuple
-
-from transformers import (
-    AutoTokenizer,
-    AutoConfig,
-)
-
-from text_generation_server.models import Seq2SeqLM
-from text_generation_server.models.custom_modeling.t5_modeling import (
-    T5ForConditionalGeneration,
-)
-from text_generation_server.utils import (
-    initialize_torch_distributed,
-    weight_files,
-    Weights,
-)
-
-
-class T5Sharded(Seq2SeqLM):
-    def __init__(
-        self,
-        model_id: str,
-        revision: Optional[str] = None,
-        quantize: Optional[str] = None,
-        speculator: Optional[str] = None,
-        dtype: Optional[torch.dtype] = None,
-        trust_remote_code: bool = False,
-    ):
-        self.process_group, rank, world_size = initialize_torch_distributed()
-        if torch.cuda.is_available():
-            device = torch.device(f"cuda:{rank}")
-            dtype = torch.float16 if dtype is None else dtype
-        else:
-            device = torch.device("cpu")
-            dtype = torch.float32 if dtype is None else dtype
-
-        config = AutoConfig.from_pretrained(
-            model_id,
-            revision=revision,
-            trust_remote_code=trust_remote_code,
-        )
-        config.quantize = quantize
-        config.speculator = speculator
-
-        tokenizer = AutoTokenizer.from_pretrained(
-            model_id,
-            revision=revision,
-            padding_side="left",
-            truncation_side="left",
-            trust_remote_code=trust_remote_code,
-        )
-        tokenizer.bos_token_id = config.decoder_start_token_id
-
-        torch.distributed.barrier(group=self.process_group)
-        filenames = weight_files(model_id, revision=revision, extension=".safetensors")
-        weights = Weights(
-            filenames,
-            device=device,
-            dtype=dtype,
-            process_group=self.process_group,
-            aliases={
-                "shared.weight": [
-                    "encoder.embed_tokens.weight",
-                    "decoder.embed_tokens.weight",
-                ]
-            },
-        )
-
-        model = T5ForConditionalGeneration(config, weights)
-
-        torch.distributed.barrier(group=self.process_group)
-        super(Seq2SeqLM, self).__init__(
-            model=model,
-            tokenizer=tokenizer,
-            requires_padding=True,
-            dtype=dtype,
-            device=device,
-            rank=rank,
-            world_size=world_size,
-        )
-
-    def forward(
-        self,
-        input_ids,
-        attention_mask,
-        decoder_input_ids,
-        decoder_attention_mask: Optional,
-        encoder_last_hidden_state: Optional,
-        past_key_values: Optional = None,
-    ) -> Tuple[
-        torch.Tensor,
-        torch.Tensor,
-        List[Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]],
-    ]:
-        # Model Forward
-        outputs, speculative_logits = self.model.forward(
-            input_ids=input_ids,
-            attention_mask=attention_mask,
-            decoder_input_ids=decoder_input_ids,
-            decoder_attention_mask=decoder_attention_mask,
-            encoder_outputs=encoder_last_hidden_state,
-            past_key_values=past_key_values,
-            use_cache=True,
-        )
-
-        return (
-            outputs.logits,
-            speculative_logits,
-            outputs.encoder_last_hidden_state,
-            outputs.past_key_values,
-        )
diff --git a/server/text_generation_server/models/types.py b/server/text_generation_server/models/types.py
index 339b733b..d4e7cca7 100644
--- a/server/text_generation_server/models/types.py
+++ b/server/text_generation_server/models/types.py
@@ -1,4 +1,3 @@
-from functools import total_ordering
 import torch
 
 from abc import ABC, abstractmethod
diff --git a/server/text_generation_server/models/vlm_causal_lm.py b/server/text_generation_server/models/vlm_causal_lm.py
index 55dc7c07..5c9955c2 100644
--- a/server/text_generation_server/models/vlm_causal_lm.py
+++ b/server/text_generation_server/models/vlm_causal_lm.py
@@ -9,7 +9,7 @@ import base64
 import numpy
 from opentelemetry import trace
 from loguru import logger
-from typing import Optional, Tuple, List, Type, Dict
+from typing import Iterable, Optional, Tuple, List, Type, Dict
 import itertools
 import tempfile
 import copy
@@ -32,9 +32,6 @@ from transformers.models.llava_next.modeling_llava_next import (
 from transformers import AutoProcessor
 import text_generation_server.habana_quantization_env as hq_env
 from optimum.habana.transformers.modeling_utils import adapt_transformers_to_gaudi
-from text_generation_server.models.cache_manager import (
-    get_cache_manager,
-)
 from text_generation_server.utils import (
     HeterogeneousNextTokenChooser,
     StoppingCriteria,
@@ -71,6 +68,10 @@ from text_generation_server.utils.debug import dbg_trace
 
 tracer = trace.get_tracer(__name__)
 
+IDEFICS2_FAKE_TOKEN = "<fake_token_around_image>"
+IDEFICS2_IMAGE_TOKEN = "<image>"
+
+
 IMAGES = re.compile(r"!\[[^\]]*\]\((.*?)\s*(\"(?:.*[^\"])\")?\s*\)")
 BASE_IMAGE_TOKENS = int(os.environ.get('BASE_IMAGE_TOKENS', 2048))
 MAX_TOTAL_TOKENS = int(os.environ.get('MAX_TOTAL_TOKENS', 8192))
@@ -105,19 +106,17 @@ def split(string) -> List[Dict[str, str]]:
 
     return parts
 
-def image_text_replacement(image_input, config, image_id) -> str:
+def image_text_replacement(processor, image_input, config, image_id: int) -> str:
     if config.model_type == "idefics2":
-        # TODO technically depends on image splitting which is not implemented.
-        num_features = 320
-        return (
-            "<fake_token_around_image>"
-            + "<image>" * num_features
-            + "<fake_token_around_image>"
-        )
+        image_seq_len = 64
+        image_str = f"{IDEFICS2_FAKE_TOKEN}{IDEFICS2_IMAGE_TOKEN * image_seq_len}{IDEFICS2_FAKE_TOKEN}"
+        if processor.image_processor.do_image_splitting:
+            image_str *= 5
+        return image_str
     elif config.model_type == "llava_next":
         height, width = image_input["image_sizes"][image_id]
         num_features = get_number_of_features(height, width, config)
-
+        from loguru import logger
         return "<image>" * num_features
 
     elif config.model_type == "paligemma":
@@ -126,20 +125,35 @@ def image_text_replacement(image_input, config, image_id) -> str:
         raise RuntimeError(f"Unknown config {config.model_type} for multimodal")
 
 
+def image_text_replacement_fixup(config, text: str) -> str:
+    if config.model_type == "idefics2":
+        return text.replace(
+            f"{IDEFICS2_FAKE_TOKEN}{IDEFICS2_FAKE_TOKEN}", IDEFICS2_FAKE_TOKEN
+        )
+    return text
+
+
 def get_unpadded_features(
-    height: int, width: int, npatches: int, num_patch_height: int, num_patch_width: int
+    original_height: int,
+    original_width: int,
+    npatches: int,
+    num_patch_height: int,
+    num_patch_width: int,
 ) -> Tuple[int, int]:
     current_height = npatches * num_patch_height
     current_width = npatches * num_patch_width
 
-    aspect_ratio: float = width / height
+    aspect_ratio: float = original_width / original_height
     current_aspect_ratio: float = current_width / current_height
+
     if aspect_ratio > current_aspect_ratio:
-        new_height = (height * current_width) // width
-        current_height = new_height
+        new_height = (original_height * current_width) // original_width
+        padding = (current_height - new_height) // 2
+        current_height = current_height - (2 * padding)
     else:
-        new_width = (width * current_height) // height
-        current_width = new_width
+        new_width = (original_width * current_height) // original_height
+        padding = (current_width - new_width) // 2
+        current_width = current_width - (2 * padding)
 
     unpadded_features = current_height * current_width
     newline_features = current_height
@@ -158,7 +172,9 @@ def get_number_of_features(height: int, width: int, config) -> int:
 
     npatches = image_size // patch_size
 
-    num_patch_height, num_patch_width = get_anyres_image_grid_shape(
+    # Dimensions are intentionally swapped to be bug-compatible with
+    # upstream: https://github.com/LLaVA-VL/LLaVA-NeXT/issues/59
+    num_patch_width, num_patch_height = get_anyres_image_grid_shape(
         [height, width],
         image_grid_pinpoints,
         image_size,
@@ -172,12 +188,6 @@ def get_number_of_features(height: int, width: int, config) -> int:
     return unpadded_features + newline_features + base_features
 
 
-def load_data_uri(image_uri: str) -> Image.Image:
-    image_uri = image_uri.split(",")[-1]
-    content = base64.b64decode(image_uri)
-    image = Image.open(BytesIO(content))
-    return image
-
 class VlmCausalLMBatch(CausalLMBatch):
     pixel_values: Optional[List[torch.Tensor]]
     pixel_attention_mask: Optional[List[torch.Tensor]]
@@ -274,76 +284,99 @@ class VlmCausalLMBatch(CausalLMBatch):
             input_length=input_len,
         )
 
+
     @classmethod
-    def batch_tokenized_inputs(cls, requests, tokenizer, processor, config, is_warmup):
-        batch_inputs = []
-        image_inputs = []
-        max_truncation = 0
+    def batch_tokenized_inputs(
+        cls, requests: Iterable[generate_pb2.Request], tokenizer, processor, config, is_warmup
+    ):
+        # Process images first. We need all of them so that the processor
+        # can make the image splits the same size. And we need the final
+        # sizes to insert correct number of image tokens.
+        images = []
         for r in requests:
-            chunks = split(r.inputs)
-            full_text = ""
-            image_id = 0
-            for chunk in chunks:
-                if chunk["type"] == "text":
-                    full_text += chunk["content"]
-                elif chunk["type"] == "image":
-                    image = chunk["content"]
-                    # Should never receive URLs anymore, processing should be done
-                    # On the rust layer.
-                    # This avoid making n queries per TP
-                    # if image.startswith("https://") or image.startswith("http://"):
-                    #     image = processor.image_processor.fetch_images(image)
-                    if image.startswith("data:"):
-                        image = load_data_uri(image)
+            for chunk in r.input_chunks.chunks:
+                chunk_type = chunk.WhichOneof("chunk")
+                if chunk_type == "text":
+                    pass
+                elif chunk_type == "image":
+                    image = Image.open(BytesIO(chunk.image.data))
+                    if config.model_type == "llava_next":
+                        images.append(image)
                     else:
-                        raise RuntimeError(
-                            "Cannot process input image not starting with data:"
-                        )
-                    image_input = processor.image_processor(image, return_tensors="pt")
-                    full_text += image_text_replacement(image_input, config, image_id)
-                    image_inputs.append(image_input)
+                        images.append([image])
                 else:
-                    raise RuntimeError(f"Invalid chunk type {chunk['type']}")
+                    raise RuntimeError(f"Invalid chunk type {chunk_type}")
+
+        image_inputs = None
+        if images:
+            image_inputs = processor.image_processor(images, return_tensors="pt")
+
+        batch_inputs = []
+        max_truncation = 0
+        image_id = 0
+        for r in requests:
+            full_text = ""
+            for chunk in r.input_chunks.chunks:
+                chunk_type = chunk.WhichOneof("chunk")
+                if chunk_type == "text":
+                    full_text += chunk.text
+                elif chunk_type == "image":
+                    full_text += image_text_replacement(
+                        processor, image_inputs, config, image_id
+                    )
+                    image_id += 1
+            full_text = image_text_replacement_fixup(config, full_text)
+
             batch_inputs.append(full_text)
             max_truncation = max(max_truncation, r.truncate)
 
+        missing_inputs = 0
+        dummy_images = None
         if is_warmup is False:
             new_bs = round_up(PREFILL_WARMUP_BATCH_SIZE_LIST, len(requests))
             missing_inputs = new_bs - len(requests)
-            dummy_images = []
-            dummy_inputs = []
-            if len(batch_inputs) > 0 and len(image_inputs) > 0:
-                dummy_inputs = [batch_inputs[0]] * missing_inputs
-                dummy_images = [image_inputs[0]] * missing_inputs
-            image_inputs += dummy_images
-            batch_inputs += dummy_inputs
+            if missing_inputs > 0:
+                dummy_inputs = []
+                if len(batch_inputs) > 0:
+                    dummy_inputs = [batch_inputs[0]] * missing_inputs
+
+                batch_inputs += dummy_inputs
 
         batch_tokenized_inputs = tokenizer(
             batch_inputs,
             truncation=True,
             max_length=max_truncation,
+            add_special_tokens=not config.model_type == "paligemma",
             return_tensors="pt",
             padding="longest",
             return_token_type_ids=False,
         )
-        if image_inputs:
-            image_input = image_inputs[0]
+
+        if missing_inputs > 0 and image_inputs is not None:
+            dummy_shape = list(image_inputs['pixel_values'].shape)
+            dummy_shape[0] = missing_inputs
+            dummy_images = torch.rand(dummy_shape)
             new_image_inputs = {
                 "pixel_values": torch.cat(
-                    [img["pixel_values"] for img in image_inputs], dim=0
+                    (image_inputs['pixel_values'], dummy_images), dim=0
                 ),
             }
-            if "pixel_attention_mask" in image_input:
+            if "pixel_attention_mask" in image_inputs:
+                dummy_shape = list(image_inputs['pixel_attention_mask'].shape)
+                dummy_shape[0] = missing_inputs
+                dummy_attention = torch.zeros(dummy_shape)
                 new_image_inputs["pixel_attention_mask"] = torch.cat(
-                    [img["pixel_attention_mask"] for img in image_inputs], dim=0
+                    (image_inputs["pixel_attention_mask"], dummy_attention), dim=0
                 )
-            if "image_sizes" in image_input:
+            if "image_sizes" in image_inputs:
+                dummy_shape = list(list(image_inputs['image_sizes'])[0])
+                dummy_shape = missing_inputs*[dummy_shape]
+                dummy_sizes = torch.IntTensor(dummy_shape)
                 new_image_inputs["image_sizes"] = torch.cat(
-                    [img["image_sizes"] for img in image_inputs], dim=0
+                    (image_inputs["image_sizes"], dummy_sizes), dim=0
                 )
             image_inputs = new_image_inputs
-        else:
-            image_inputs = None
+
         return batch_tokenized_inputs, image_inputs
 
     @classmethod
@@ -498,6 +531,7 @@ class VlmCausalLM(Model):
         processor_kwargs=None,
         batch_class=VlmCausalLMBatch,
         revision,
+        quantize: Optional[str] = None,
         dtype,
         trust_remote_code: bool,
         **kwargs,
@@ -513,6 +547,7 @@ class VlmCausalLM(Model):
         )
         self.batch_class = batch_class
         self.prev_bs = 0
+        self.quantize = quantize
 
         # Create tokenizer
         tokenizer = AutoTokenizer.from_pretrained(
@@ -522,6 +557,7 @@ class VlmCausalLM(Model):
             truncation_side="left",
             trust_remote_code=trust_remote_code,
         )
+        make_tokenizer_optional(tokenizer)
 
         # Create model
         world_size = int(os.getenv("WORLD_SIZE", "1"))
@@ -580,35 +616,42 @@ class VlmCausalLM(Model):
             if model.config.pad_token_id is not None:
                 tokenizer.pad_token_id = model.config.pad_token_id
             elif model.config.eos_token_id is not None:
-                tokenizer.pad_token_id = model.config.eos_token_id
+                if isinstance(model.config.eos_token_id, int):
+                    tokenizer.pad_token_id = model.config.eos_token_id
+                elif isinstance(model.config.eos_token_id, list):
+                    tokenizer.pad_token_id = model.config.eos_token_id[0]
+                else:
+                    raise ValueError(
+                        f"{type(model.config.eos_token_id)} type of eos_token_id in the model's config is not supported for tokenizer.pad_token_id"
+                    )
             elif tokenizer.eos_token_id is not None:
                 tokenizer.pad_token_id = tokenizer.eos_token_id
             else:
                 tokenizer.add_special_tokens({"pad_token": "[PAD]"})
 
-        kwargs = {
+        self.kwargs = {
             "use_cache": True,
             "return_dict": True,
         }
 
-        if model.config.model_type in ["llama", "mistral", "llava_next"]:
-            kwargs["attn_softmax_bf16"] = True
-            kwargs["trim_logits"] = True
+        if model.config.model_type in ["llava_next"]:
+            self.kwargs["attn_softmax_bf16"] = True
+            self.kwargs["trim_logits"] = True
 
             if os.getenv("USE_FLASH_ATTENTION", "false").lower() == "true":
-                kwargs["use_flash_attention"] = True
+                self.kwargs["use_flash_attention"] = True
             if os.getenv("FLASH_ATTENTION_RECOMPUTE", "false").lower() == "true":
-                kwargs["flash_attention_recompute"] = True
+                self.kwargs["flash_attention_recompute"] = True
 
         self.speculate = get_speculate()
         super(VlmCausalLM, self).__init__(
+            model_id=model_id,
             model=model,
             tokenizer=tokenizer,
             requires_padding=True,
             dtype=dtype,
             device=device,
             rank=rank,
-            kwargs=kwargs,
         )
 
         # Create profiler
@@ -1059,7 +1102,7 @@ class VlmCausalLM(Model):
         )
 
     def generate_warmup_batch(self, request, seq_len, batch_size, is_warmup):
-        batch = copy.deepcopy(request.batches[0])
+        batch = copy.deepcopy(request.batch)
         for req in batch.requests:
             req.truncate = seq_len
 
@@ -1070,20 +1113,20 @@ class VlmCausalLM(Model):
 
     def warmup(self, request) -> None:
         is_warmup = True
-        batches = [self.batch_from_pb(batch, is_warmup) for batch in request.batches]
+        batch = self.batch_from_pb(request.batch, is_warmup)
 
         try:
             # max prefill batch size warmup
-            _, prefill_batch, _ = self.generate_token([batches[0]], is_warmup)
+            _, prefill_batch, _ = self.generate_token([batch], is_warmup)
         except:
             raise RuntimeError(
-                f"Not enough memory to handle {len(batches[0].input_ids)} prefill tokens. "
+                f"Not enough memory to handle {len(batch.input_ids)} prefill tokens. "
                 f"You need to decrease `--max-batch-prefill-tokens`"
             )
 
         global BASE_IMAGE_TOKENS, MAX_TOTAL_TOKENS, MAX_BATCH_TOTAL_TOKENS, PREFILL_WARMUP_BATCH_SIZE_LIST, PREFILL_WARMUP_SEQLEN_LIST, DECODE_WARMUP_BATCH_SIZE_LIST
-        max_input_length =  batches[0].input_ids.shape[1]
-        max_prefill_batch_size = batches[0].input_ids.shape[0]
+        max_input_length =  batch.input_ids.shape[1]
+        max_prefill_batch_size = batch.input_ids.shape[0]
         PREFILL_WARMUP_BATCH_SIZE_LIST = []
         batch_size = 1
         while batch_size <= max_prefill_batch_size:
diff --git a/server/text_generation_server/server.py b/server/text_generation_server/server.py
index a4dc4a4c..bc458a91 100644
--- a/server/text_generation_server/server.py
+++ b/server/text_generation_server/server.py
@@ -16,10 +16,12 @@ from typing import List, Optional
 
 from text_generation_server.cache import Cache
 from text_generation_server.interceptor import ExceptionInterceptor
-from text_generation_server.models import Model, get_model
+from text_generation_server.models import Model, get_model_with_lora_adapters
 from text_generation_server.pb import generate_pb2_grpc, generate_pb2
 from text_generation_server.tracing import UDSOpenTelemetryAioServerInterceptor
 from text_generation_server.models.globals import set_model_id
+from text_generation_server.models.globals import set_adapter_to_index
+from text_generation_server.utils.adapter import AdapterInfo
 
 try:
     from text_generation_server.models.pali_gemma import PaliGemmaBatch
@@ -28,7 +30,11 @@ try:
     )
     from text_generation_server.models.idefics_causal_lm import IdeficsCausalLMBatch
 
-    VLM_BATCH_TYPES = {PaliGemmaBatch, VlmCausalLMBatch, IdeficsCausalLMBatch}
+    VLM_BATCH_TYPES = {
+        PaliGemmaBatch,
+        VlmCausalLMBatch,
+        IdeficsCausalLMBatch,
+    }
 except (ImportError, NotImplementedError):
     # These imports can fail on CPU/Non flash.
     VLM_BATCH_TYPES = set()
@@ -56,6 +62,8 @@ class TextGenerationService(generate_pb2_grpc.TextGenerationServiceServicer):
     ):
         self.cache = cache
         self.model = model
+        # Quantize is resolved during model loading
+        self.quantize = model.quantize
         self.server_urls = server_urls
         # For some reason, inference_mode does not work well with GLOO which we use on CPU
         # TODO: The inferecemode set messes up the autograd op dispatch. And results in aten::matmul
@@ -93,19 +101,9 @@ class TextGenerationService(generate_pb2_grpc.TextGenerationServiceServicer):
         return generate_pb2.FilterBatchResponse(batch=filtered_batch.to_pb())
 
     async def Warmup(self, request, context):
-        def batch_from_pb(batch):
-            return self.model.batch_type.from_pb(
-                batch, self.model.tokenizer, self.model.dtype, self.model.device
-            )
-
-        if self.model.batch_type in VLM_BATCH_TYPES :
-            max_supported_total_tokens = self.model.warmup(request)
-            return generate_pb2.WarmupResponse(max_supported_total_tokens=max_supported_total_tokens)
-        else:
-            batches = [batch_from_pb(batch) for batch in request.batches]
-            self.model.warmup(batches)
-            return generate_pb2.WarmupResponse()
 
+        max_supported_total_tokens = self.model.warmup(request)
+        return generate_pb2.WarmupResponse(max_supported_total_tokens=max_supported_total_tokens)
 
     async def Prefill(self, request, context):
         start = time.time_ns()
@@ -124,7 +122,7 @@ class TextGenerationService(generate_pb2_grpc.TextGenerationServiceServicer):
             batch = self.model.batch_type.from_pb(
                 request.batch, self.model.tokenizer, self.model.dtype, self.model.device
             )
-     
+
         generations, next_batch, timings = self.model.generate_token([batch])
         self.cache.set(next_batch)
 
@@ -157,7 +155,7 @@ class TextGenerationService(generate_pb2_grpc.TextGenerationServiceServicer):
         return generate_pb2.DecodeResponse(
             generations=[generation.to_pb() for generation in generations],
             batch=next_batch.to_pb() if next_batch else None,
-            concat_ns=None, # TODO: measure concat time
+            concat_ns=None,
             forward_ns=timings[0],
             decode_ns=timings[1],
             total_ns=time.time_ns() - start,
@@ -166,29 +164,22 @@ class TextGenerationService(generate_pb2_grpc.TextGenerationServiceServicer):
 
 def serve(
     model_id: str,
+    lora_adapters: Optional[List[AdapterInfo]],
     revision: Optional[str],
     sharded: bool,
+    quantize: Optional[str],
     speculate: Optional[int],
     dtype: Optional[str],
     trust_remote_code: bool,
     uds_path: Path,
+    max_input_tokens: int,
 ):
-    # Remove default handler
-    #logger.remove()
-    logger.add(
-        sys.stdout,
-        format="{message}",
-        filter="text_generation_server",
-        level="INFO",
-        serialize=False,
-        backtrace=True,
-        diagnose=False,
-    )
-
     async def serve_inner(
         model_id: str,
+        lora_adapters: Optional[List[AdapterInfo]],
         revision: Optional[str],
         sharded: bool = False,
+        quantize: Optional[str] = None,
         speculate: Optional[int] = None,
         dtype: Optional[str] = None,
         trust_remote_code: bool = False,
@@ -197,6 +188,7 @@ def serve(
             logger.warning(f"Current Synapse version is lower than the minimum version supported: {MIN_TGI_GAUDI_SYNAPSE_VERSION}, this could result in failures")
 
         unix_socket_template = "unix://{}-{}"
+        adapter_to_index = {}
         logger.info("Server:server_inner: sharded ={}".format(sharded))
 
         if sharded:
@@ -218,22 +210,33 @@ def serve(
         if revision == "None":
             revision = None
         try:
-            model = get_model(
+            model = get_model_with_lora_adapters(
                 model_id,
+                lora_adapters,
                 revision,
+                sharded,
+                quantize,
                 speculate,
                 data_type,
-                trust_remote_code
+                trust_remote_code,
+                max_input_tokens,
+                adapter_to_index,
             )
+
         except Exception:
             logger.exception("Error when initializing model")
             raise
 
+        set_adapter_to_index(adapter_to_index)
         server = aio.server(
             interceptors=[
                 ExceptionInterceptor(),
                 UDSOpenTelemetryAioServerInterceptor(),
-            ]
+            ],
+            options=[
+                # Set the maximum possible message length: i32::MAX
+                ("grpc.max_receive_message_length", (1 << 31) - 1)
+            ],
         )
         generate_pb2_grpc.add_TextGenerationServiceServicer_to_server(
             TextGenerationService(model, Cache(), server_urls), server
@@ -255,6 +258,13 @@ def serve(
     set_model_id(model_id)
     asyncio.run(
         serve_inner(
-            model_id, revision, sharded, speculate, dtype, trust_remote_code
+            model_id,
+            lora_adapters,
+            revision,
+            sharded,
+            quantize,
+            speculate,
+            dtype,
+            trust_remote_code,
         )
     )
diff --git a/server/text_generation_server/tgi_service.py b/server/text_generation_server/tgi_service.py
index f88c8c8b..f0f13126 100644
--- a/server/text_generation_server/tgi_service.py
+++ b/server/text_generation_server/tgi_service.py
@@ -4,6 +4,8 @@ from loguru import logger
 import sys
 from text_generation_server import server
 import argparse
+from typing import List
+from text_generation_server.utils.adapter import parse_lora_adapters
 
 
 def main(args):
@@ -13,14 +15,18 @@ def main(args):
             args.model_id, args.revision, args.sharded, args.speculate, args.dtype, args.trust_remote_code, args.uds_path
         )
     )
+    lora_adapters = parse_lora_adapters(os.getenv("LORA_ADAPTERS"))
     server.serve(
         model_id=args.model_id,
+        lora_adapters=lora_adapters,
         revision=args.revision,
         sharded=args.sharded,
+        quantize=args.quantize,
         speculate=args.speculate,
         dtype=args.dtype,
         trust_remote_code=args.trust_remote_code,
         uds_path=args.uds_path,
+        max_input_tokens=args.max_input_tokens
     )
 
 
@@ -33,5 +39,7 @@ if __name__ == "__main__":
     parser.add_argument("--dtype", type=str)
     parser.add_argument("--trust_remote_code", type=bool)
     parser.add_argument("--uds_path", type=Path)
+    parser.add_argument("--quantize", type=str)
+    parser.add_argument("--max_input_tokens", type=int)
     args = parser.parse_args()
     main(args)
diff --git a/server/text_generation_server/tracing.py b/server/text_generation_server/tracing.py
index bf03c379..bc7a04ee 100644
--- a/server/text_generation_server/tracing.py
+++ b/server/text_generation_server/tracing.py
@@ -54,10 +54,8 @@ class UDSOpenTelemetryAioServerInterceptor(OpenTelemetryAioServerInterceptor):
         )
 
 
-def setup_tracing(shard: int, otlp_endpoint: str):
-    resource = Resource.create(
-        attributes={"service.name": f"text-generation-inference.server-{shard}"}
-    )
+def setup_tracing(otlp_service_name: str, otlp_endpoint: str):
+    resource = Resource.create(attributes={"service.name": otlp_service_name})
     span_exporter = OTLPSpanExporter(endpoint=otlp_endpoint, insecure=True)
     span_processor = BatchSpanProcessor(span_exporter)
 
diff --git a/server/text_generation_server/utils/adapter.py b/server/text_generation_server/utils/adapter.py
new file mode 100644
index 00000000..2b61f9bb
--- /dev/null
+++ b/server/text_generation_server/utils/adapter.py
@@ -0,0 +1,320 @@
+# Origin:   https://github.com/predibase/lorax
+# Path:     lorax/server/lorax_server/utils/adapter.py
+# License:  Apache License Version 2.0, January 2004
+
+import warnings
+import re
+from dataclasses import dataclass
+from functools import lru_cache
+from typing import TYPE_CHECKING, Set, Tuple, Optional, List
+
+from safetensors.torch import load_file
+from transformers import AutoConfig, AutoTokenizer, PreTrainedTokenizer
+
+from text_generation_server.utils.merges.strategies import merge_adapters
+
+from text_generation_server.utils import hub
+from text_generation_server.adapters.lora import LoraConfig
+
+
+if TYPE_CHECKING:
+    from text_generation_server.adapters.config import AdapterConfig, ModuleMap
+
+
+BASE_MODEL_ADAPTER_ID = "__base_model__"
+
+
+@dataclass
+class AdapterInfo:
+    id: str
+    path: Optional[str]
+    revision: Optional[str] = None
+
+
+@dataclass
+class AdapterParameters:
+    adapter_info: Tuple[AdapterInfo]
+    weights: Tuple[float]
+    merge_strategy: NotImplemented
+    density: float
+    majority_sign_method: NotImplemented
+
+
+@dataclass
+class AdapterSource:
+    adapter_id: str
+    model_id: str
+    revision: str
+
+
+def parse_lora_adapters(lora_adapters: Optional[str]) -> List[AdapterInfo]:
+    if not lora_adapters:
+        return []
+
+    adapter_list = []
+    for adapter in lora_adapters.split(","):
+        adapter = adapter.strip()
+        if adapter.count("=") > 1 or adapter.count("@") > 1:
+            raise ValueError(f"Invalid LoRA adapter format: {adapter}")
+        match = re.match(r"^([^=@]+)(?:=([^@]+))?(?:@(.+))?$", adapter)
+
+        if match:
+            adapter_id, path, revision = match.groups()
+            adapter_list.append(
+                AdapterInfo(id=adapter_id, path=path, revision=revision)
+            )
+        else:
+            raise ValueError(f"Invalid LoRA adapter format: {adapter}")
+    return adapter_list
+
+
+def load_and_merge_adapters(
+    model_id: str,
+    adapter_parameters: AdapterParameters,
+    adapter_index: int,
+    weight_names: Tuple[str],
+    trust_remote_code: bool = False,
+) -> Tuple["ModuleMap", "AdapterConfig", Set[str], PreTrainedTokenizer]:
+    if len(adapter_parameters.adapter_info) == 1:
+        adapter = next(iter(adapter_parameters.adapter_info))
+        return load_module_map(
+            model_id,
+            adapter.revision,
+            adapter.id,
+            adapter.path,
+            weight_names,
+            trust_remote_code,
+        )
+
+    adapter_params = AdapterParametersContainer(adapter_parameters, adapter_index)
+    return _load_and_merge(
+        model_id,
+        adapter_params,
+        weight_names,
+        trust_remote_code,
+    )
+
+
+@dataclass
+class AdapterParametersContainer:
+    adapter_parameters: AdapterParameters
+    adapter_index: int
+
+    def __hash__(self) -> int:
+        return self.adapter_index
+
+
+@lru_cache(maxsize=32)
+def _load_and_merge(
+    model_id: str,
+    adapter_params: AdapterParametersContainer,
+    weight_names: Tuple[str],
+    trust_remote_code: bool = False,
+) -> Tuple["ModuleMap", "AdapterConfig", Set[str], PreTrainedTokenizer]:
+    params = adapter_params.adapter_parameters
+
+    adapters_to_merge = []
+    merged_weight_names = set()
+    tokenizer = None
+    for adapter in params.adapter_info:
+        if adapter.id == BASE_MODEL_ADAPTER_ID:
+            raise ValueError("Base model adapter cannot be merged.")
+
+        module_map, adapter_config, adapter_weight_names, adapter_tokenizer = (
+            load_module_map(
+                model_id,
+                adapter.revision,
+                adapter.id,
+                adapter.path,
+                weight_names,
+                trust_remote_code,
+            )
+        )
+
+        adapters_to_merge.append((module_map, adapter_config))
+        merged_weight_names = merged_weight_names.union(adapter_weight_names)
+        if tokenizer is None:
+            tokenizer = adapter_tokenizer
+
+    if len(adapters_to_merge) == 0:
+        raise ValueError("No adapters to merge.")
+
+    module_map, adapter_config = merge_adapters(adapters_to_merge, params)
+    return module_map, adapter_config, merged_weight_names, tokenizer
+
+
+def check_architectures(
+    model_id: str,
+    adapter_id: str,
+    adapter_config: "AdapterConfig",
+    trust_remote_code: bool = False,
+):
+    try:
+        if not adapter_config.base_model_name_or_path:
+            # Avoid execution latency caused by the network connection retrying for AutoConfig.from_pretrained(None)
+            return
+
+        expected_config = AutoConfig.from_pretrained(
+            model_id, trust_remote_code=trust_remote_code
+        )
+        model_config = AutoConfig.from_pretrained(
+            adapter_config.base_model_name_or_path, trust_remote_code=trust_remote_code
+        )
+    except Exception as e:
+        warnings.warn(
+            f"Unable to check architecture compatibility for adapter '{adapter_id}' "
+            f"against model '{model_id}'. Assuming they are compatible. Error: {e}"
+        )
+        return
+
+    if model_config.architectures == expected_config.architectures:
+        warnings.warn(
+            f"Adapter '{adapter_id}' was not trained on base model '{model_id}'. "
+            f"If you encounter issues, use --model-id '{adapter_config.base_model_name_or_path}' instead."
+        )
+    else:
+        # TODO(travis): revisit this when we support clasification heads which will not use CausalLM
+        raise ValueError(
+            f"Adapter '{adapter_id}' is not compatible with model '{model_id}'. "
+            f"Architectures differ: {model_config.architectures} != {expected_config.architectures}. "
+            f"Use --model-id '{adapter_config.base_model_name_or_path}' instead."
+        )
+
+
+@lru_cache(maxsize=128)
+def load_module_map(
+    model_id: str,
+    revision: str,
+    adapter_id: str,
+    adapter_path: Optional[str],
+    weight_names: Tuple[str],
+    trust_remote_code: bool = False,
+) -> Tuple["ModuleMap", "AdapterConfig", Set[str], PreTrainedTokenizer]:
+    adapter_config = LoraConfig.load(adapter_path or adapter_id, None)
+
+    if not adapter_path and adapter_config.base_model_name_or_path != model_id:
+        check_architectures(model_id, adapter_id, adapter_config, trust_remote_code)
+
+    adapter_filenames = (
+        hub._weight_files_from_dir(adapter_path, extension=".safetensors")
+        if adapter_path
+        else hub._cached_weight_files(
+            adapter_id, revision=revision, extension=".safetensors"
+        )
+    )
+
+    # throw an error if no adapter weights are found
+    if not adapter_filenames:
+        raise FileNotFoundError(
+            f"No adapter weights found for adapter '{adapter_id}' and revision '{revision}'."
+        )
+
+    try:
+        adapter_tokenizer = AutoTokenizer.from_pretrained(
+            adapter_config.config_path,
+            trust_remote_code=trust_remote_code,
+        )
+    except Exception:
+        # Adapter does not have a tokenizer, so fallback to base model tokenizer
+        adapter_tokenizer = None
+
+    # load adapter weights from all shards (should have relatively small memory footprint)
+    adapter_weights = {}
+    for filename in adapter_filenames:
+        adapter_weights.update(load_file(filename))
+
+    # map the model weights to the relevant adapter weights (LoRA A and B matrices)
+    module_map, adapter_weight_names = adapter_config.map_weights_for_model(
+        adapter_weights, weight_names
+    )
+    return module_map, adapter_config, adapter_weight_names, adapter_tokenizer
+
+
+def get_attn_weights(i, layer):
+    qkv = layer.self_attn.query_key_value
+    weights = {}
+
+    for k in ["q", "k", "v"]:
+        key = (i, f"{k}_proj")
+        value = (f"model.layers.{i}.self_attn.{k}_proj", qkv)
+        weights[key] = value
+
+    # also add the qkv_proj weight for the adapter
+    weights[(i, "qkv_proj")] = (
+        f"model.layers.{i}.self_attn.qkv_proj",
+        qkv,
+    )
+
+    weights[(i, "o_proj")] = (
+        f"model.layers.{i}.self_attn.o_proj",
+        layer.self_attn.o_proj,
+    )
+
+    return weights
+
+
+def get_mlp_weights(i, layer):
+    weights = {}
+    if hasattr(layer, "mlp"):
+        mlp = layer.mlp
+        if hasattr(mlp, "gate_up_proj"):
+            # handle combined gate_up_proj (e.g., for some LLaMA variants)
+            weights.update(
+                {
+                    (i, "gate_proj"): (
+                        f"model.layers.{i}.mlp.gate_proj",
+                        mlp.gate_up_proj,
+                    ),
+                    (i, "up_proj"): (f"model.layers.{i}.mlp.up_proj", mlp.gate_up_proj),
+                }
+            )
+        else:
+            # handle separate gate_proj, up_proj, and down_proj (e.g., for Gemma)
+            if hasattr(mlp, "gate_proj"):
+                weights[(i, "gate_proj")] = (
+                    f"model.layers.{i}.mlp.gate_proj",
+                    mlp.gate_proj,
+                )
+            if hasattr(mlp, "up_proj"):
+                weights[(i, "up_proj")] = (f"model.layers.{i}.mlp.up_proj", mlp.up_proj)
+
+        if hasattr(mlp, "down_proj"):
+            weights[(i, "down_proj")] = (
+                f"model.layers.{i}.mlp.down_proj",
+                mlp.down_proj,
+            )
+
+    return weights
+
+
+# build_layer_weight_lookup creates a mapping of model layers to their corresponding
+# weight tensors and paths. It builds a dictionary that maps layer identifiers to tuples
+# containing the weight tensor path and the actual layer object. This mapping is needed
+# for the lora adapter to know which weights to update when applying the adapter.
+def build_layer_weight_lookup(model):
+    if hasattr(model, "language_model"):
+        m = model.language_model.model
+    elif hasattr(model, "text_model"):
+        m = model.text_model.model
+    else:
+        m = model.model
+
+    layer_weights = {}
+
+    for i, layer in enumerate(m.layers):
+        attn_weights = get_attn_weights(i, layer)
+        mlp_weights = get_mlp_weights(i, layer)
+
+        layer_weights.update(attn_weights)
+        layer_weights.update(mlp_weights)
+
+    lm_head = None
+    if hasattr(m, "lm_head"):
+        lm_head = m.lm_head
+    elif hasattr(model, "lm_head"):
+        lm_head = model.lm_head
+
+    if lm_head:
+        layer_weights[(0, "lm_head")] = ("lm_head", lm_head)
+
+    return layer_weights
diff --git a/server/text_generation_server/utils/chunks.py b/server/text_generation_server/utils/chunks.py
new file mode 100644
index 00000000..73962ea3
--- /dev/null
+++ b/server/text_generation_server/utils/chunks.py
@@ -0,0 +1,27 @@
+from typing import Iterable
+
+from loguru import logger
+
+from text_generation_server.pb import generate_pb2
+
+
+def concat_text_chunks(chunks: Iterable[generate_pb2.InputChunk]) -> str:
+    """
+    Concatenate text in text chunks. Non-text chunks are dropped.
+    """
+    text = None
+    for chunk in chunks:
+        chunk_type = chunk.WhichOneof("chunk")
+        if chunk_type == "text":
+            if text is None:
+                text = chunk.text
+            else:
+                raise NotImplementedError("Request contained more than one text chunk")
+        else:
+            # We cannot reject this, e.g. warmup sends an image chunk.
+            logger.debug(f"Encountered non-text chunk type {chunk_type}")
+
+    if text is None:
+        raise NotImplementedError("Request without a text chunk")
+
+    return text
diff --git a/server/text_generation_server/utils/flash_attn.py b/server/text_generation_server/utils/flash_attn.py
deleted file mode 100644
index 4f5cf10b..00000000
--- a/server/text_generation_server/utils/flash_attn.py
+++ /dev/null
@@ -1,293 +0,0 @@
-import os
-import torch
-
-from loguru import logger
-import math
-
-from text_generation_server.utils.import_utils import SYSTEM
-
-if SYSTEM != "xpu":
-    from text_generation_server.utils.flash_attn_triton import triton_attention
-
-if os.getenv("USE_FLASH_ATTENTION", "").lower() == "false":
-    raise ImportError("`USE_FLASH_ATTENTION` is false.")
-HAS_FLASH_ATTN = False
-HAS_FLASH_ATTN_V2_CUDA = False
-HAS_FLASH_ATTN_V2_ROCM = False
-ROCM_USE_FLASH_ATTN_V2_CK = False
-ROCM_USE_FLASH_ATTN_V2_TRITON = False
-
-
-if SYSTEM in {"cuda", "rocm"}:
-    if not torch.cuda.is_available():
-        raise ImportError("CUDA is not available")
-
-    major, minor = torch.cuda.get_device_capability()
-    is_sm75 = major == 7 and minor == 5
-    is_sm8x = major == 8 and minor >= 0
-    is_sm90 = major == 9 and minor == 0
-    is_sm94 = major == 9 and minor == 4
-
-    if SYSTEM == "rocm":
-        if (
-            os.getenv("ROCM_USE_FLASH_ATTN_V2_TRITON", "").lower() == "true"
-            or os.getenv("ROCM_USE_FLASH_ATTN_V2_TRITON", "0") == "1"
-        ):
-            ROCM_USE_FLASH_ATTN_V2_TRITON = True
-            logger.info("ROCm: using Flash Attention 2 Triton implementation.")
-        else:
-            ROCM_USE_FLASH_ATTN_V2_CK = True
-            logger.info(
-                "ROCm: using Flash Attention 2 Composable Kernel implementation."
-            )
-
-    try:
-        try:
-            import flash_attn_2_cuda
-        except ImportError:
-            architecture_suffix = f"-{SYSTEM}"
-            raise ImportError(
-                "Flash Attention V2 is not installed.\n"
-                "Use the official Docker image (ghcr.io/huggingface/text-generation-inference:latest) "
-                f"or install flash attention v2 with `cd server && make install install-flash-attention-v2{architecture_suffix}`"
-            )
-        if SYSTEM == "cuda" and not (is_sm8x or is_sm90):
-            raise ImportError(
-                f"GPU with CUDA capability {major} {minor} is not supported for "
-                "Flash Attention V2"
-            )
-        elif SYSTEM == "rocm" and not (is_sm8x or is_sm90 or is_sm94):
-            raise ImportError(
-                f"AMD GPU with compute capability {major} {minor} is not supported for "
-                "Flash Attention V2"
-            )
-        HAS_FLASH_ATTN_V2_CUDA = SYSTEM == "cuda"
-        HAS_FLASH_ATTN_V2_ROCM = SYSTEM == "rocm"
-    except ImportError as e:
-        try:
-            import flash_attn_cuda
-        except ImportError:
-            raise ImportError(
-                "Flash Attention is not installed.\n"
-                "Use the official Docker image (ghcr.io/huggingface/text-generation-inference:latest) "
-                "or install flash attention with `cd server && make install install-flash-attention`"
-            ) from e
-
-        if SYSTEM == "cuda" and not (is_sm75 or is_sm8x or is_sm90):
-            raise ImportError(
-                f"GPU with CUDA capability {major} {minor} is not supported"
-            ) from e
-        elif SYSTEM == "rocm":
-            for idx in range(torch.cuda.device_count()):
-                if "MI210" not in torch.cuda.get_device_name(
-                    idx
-                ) and "MI250" not in torch.cuda.get_device_name(idx):
-                    raise ImportError(
-                        f"AMD GPU {torch.cuda.get_device_name(idx)} does not support flash-attention"
-                    )
-
-        logger.warning(f"Unable to use Flash Attention V2: {e}")
-        HAS_FLASH_ATTN = True
-
-if SYSTEM == "xpu":
-    import intel_extension_for_pytorch as ipex
-
-    def attention(
-        q,
-        k,
-        v,
-        out,
-        cu_seqlens,
-        max_s,
-        softmax_scale,
-        window_size_left=-1,
-    ):
-        if window_size_left <= 0 and window_size_left != -1:
-            raise ValueError("`window_size_left` must be > 0 or -1")
-
-        if window_size_left != -1:
-            raise ValueError(
-                f"XPU version of Flash Attention does not support window attention (window_size_left != -1, got window_size_left={window_size_left})."
-            )
-        return ipex.llm.functional.varlen_attention(
-            q,
-            k,
-            v,
-            out,
-            cu_seqlens,
-            cu_seqlens,
-            max_s,
-            max_s,
-            0.0,
-            softmax_scale,
-            False,
-            True,
-            False,
-            None,
-        )
-
-elif HAS_FLASH_ATTN_V2_CUDA:
-
-    def attention(
-        q,
-        k,
-        v,
-        out,
-        cu_seqlens,
-        max_s,
-        softmax_scale,
-        window_size_left=-1,
-        causal=True,
-    ):
-        if window_size_left <= 0 and window_size_left != -1:
-            raise ValueError("`window_size_left` must be > 0 or -1")
-        return flash_attn_2_cuda.varlen_fwd(
-            q,
-            k,
-            v,
-            out,
-            cu_seqlens,
-            cu_seqlens,
-            None,
-            None,
-            None,
-            max_s,
-            max_s,
-            0.0,
-            softmax_scale,
-            False,
-            causal,
-            window_size_left,
-            0,
-            False,
-            None,
-        )
-
-elif HAS_FLASH_ATTN_V2_ROCM and ROCM_USE_FLASH_ATTN_V2_CK:
-
-    def attention(
-        q,
-        k,
-        v,
-        out,
-        cu_seqlens,
-        max_s,
-        softmax_scale,
-        window_size_left=-1,
-        causal=True,
-    ):
-        if window_size_left <= 0 and window_size_left != -1:
-            raise ValueError("`window_size_left` must be > 0 or -1")
-        if window_size_left != -1:
-            raise ValueError(
-                f"RoCm version of Flash Attention v2 does not support window attention (window_size_left != -1, got window_size_left={window_size_left})."
-            )
-
-        # RoCm flash API does not take the window_size_left and window_size_right arguments.
-        return flash_attn_2_cuda.varlen_fwd(
-            q,
-            k,
-            v,
-            out,
-            cu_seqlens,
-            cu_seqlens,
-            max_s,
-            max_s,
-            0.0,
-            softmax_scale,
-            False,
-            causal,
-            False,
-            None,
-        )
-
-elif HAS_FLASH_ATTN_V2_ROCM and ROCM_USE_FLASH_ATTN_V2_TRITON:
-
-    def attention(
-        q,
-        k,
-        v,
-        out,
-        cu_seqlens,
-        max_s,
-        softmax_scale,
-        window_size_left=-1,
-        causal=True,
-    ):
-        output, _ = triton_attention(
-            q,
-            k,
-            v,
-            out,
-            cu_seqlens,
-            cu_seqlens,
-            max_s,
-            max_s,
-            causal,
-            softmax_scale,
-        )
-        return output
-
-elif HAS_FLASH_ATTN:
-
-    def attention(
-        q,
-        k,
-        v,
-        out,
-        cu_seqlens,
-        max_s,
-        softmax_scale,
-        window_size_left=-1,
-    ):
-        if window_size_left != -1:
-            raise NotImplementedError(
-                "window_size_left is only available with flash attn v2"
-            )
-
-        # Flash attention v1 requires q, k and v to have the same number of heads
-        if k.shape[1] != q.shape[1]:
-            # MQA expand
-            if k.shape[1] == 1:
-                k = k.expand(-1, q.shape[1], -1)
-            # Grouped attention reshape
-            else:
-                original_shape = k.shape
-                k = (
-                    k.unsqueeze(2)
-                    .expand(-1, -1, q.shape[1] // k.shape[1], -1)
-                    .reshape(original_shape[0], -1, original_shape[2])
-                )
-        if v.shape[1] != q.shape[1]:
-            # MQA expand
-            if v.shape[1] == 1:
-                v = v.expand(-1, q.shape[1], -1)
-            # Grouped attention reshape
-            else:
-                original_shape = v.shape
-                v = (
-                    v.unsqueeze(2)
-                    .expand(-1, -1, q.shape[1] // v.shape[1], -1)
-                    .reshape(original_shape[0], -1, original_shape[2])
-                )
-
-        return flash_attn_cuda.fwd(
-            q,
-            k,
-            v,
-            out,
-            cu_seqlens,
-            cu_seqlens,
-            max_s,
-            max_s,
-            0.0,
-            softmax_scale,
-            False,
-            True,
-            False,
-            0,
-            None,
-        )
-
-else:
-    raise NotImplementedError("flash attention is not installed")
diff --git a/server/text_generation_server/utils/gptq/quant_linear.py b/server/text_generation_server/utils/gptq/quant_linear.py
deleted file mode 100644
index a832f755..00000000
--- a/server/text_generation_server/utils/gptq/quant_linear.py
+++ /dev/null
@@ -1,359 +0,0 @@
-import math
-import numpy as np
-import torch
-import torch.nn as nn
-from torch.cuda.amp import custom_bwd, custom_fwd
-
-try:
-    import triton
-    import triton.language as tl
-    from . import custom_autotune
-
-    # code based https://github.com/fpgaminer/GPTQ-triton
-    @custom_autotune.autotune(
-        configs=[
-            triton.Config(
-                {
-                    "BLOCK_SIZE_M": 64,
-                    "BLOCK_SIZE_N": 256,
-                    "BLOCK_SIZE_K": 32,
-                    "GROUP_SIZE_M": 8,
-                },
-                num_stages=4,
-                num_warps=4,
-            ),
-            triton.Config(
-                {
-                    "BLOCK_SIZE_M": 128,
-                    "BLOCK_SIZE_N": 128,
-                    "BLOCK_SIZE_K": 32,
-                    "GROUP_SIZE_M": 8,
-                },
-                num_stages=4,
-                num_warps=4,
-            ),
-            triton.Config(
-                {
-                    "BLOCK_SIZE_M": 64,
-                    "BLOCK_SIZE_N": 128,
-                    "BLOCK_SIZE_K": 32,
-                    "GROUP_SIZE_M": 8,
-                },
-                num_stages=4,
-                num_warps=4,
-            ),
-            triton.Config(
-                {
-                    "BLOCK_SIZE_M": 128,
-                    "BLOCK_SIZE_N": 32,
-                    "BLOCK_SIZE_K": 32,
-                    "GROUP_SIZE_M": 8,
-                },
-                num_stages=4,
-                num_warps=4,
-            ),
-            triton.Config(
-                {
-                    "BLOCK_SIZE_M": 64,
-                    "BLOCK_SIZE_N": 64,
-                    "BLOCK_SIZE_K": 32,
-                    "GROUP_SIZE_M": 8,
-                },
-                num_stages=4,
-                num_warps=4,
-            ),
-            triton.Config(
-                {
-                    "BLOCK_SIZE_M": 64,
-                    "BLOCK_SIZE_N": 128,
-                    "BLOCK_SIZE_K": 32,
-                    "GROUP_SIZE_M": 8,
-                },
-                num_stages=2,
-                num_warps=8,
-            ),
-            triton.Config(
-                {
-                    "BLOCK_SIZE_M": 64,
-                    "BLOCK_SIZE_N": 64,
-                    "BLOCK_SIZE_K": 64,
-                    "GROUP_SIZE_M": 8,
-                },
-                num_stages=3,
-                num_warps=8,
-            ),
-            triton.Config(
-                {
-                    "BLOCK_SIZE_M": 32,
-                    "BLOCK_SIZE_N": 32,
-                    "BLOCK_SIZE_K": 128,
-                    "GROUP_SIZE_M": 8,
-                },
-                num_stages=2,
-                num_warps=4,
-            ),
-        ],
-        key=["M", "N", "K"],
-        nearest_power_of_two=True,
-        prune_configs_by={
-            "early_config_prune": custom_autotune.matmul248_kernel_config_pruner,
-            "perf_model": None,
-            "top_k": None,
-        },
-    )
-    @triton.jit
-    def matmul_248_kernel(
-        a_ptr,
-        b_ptr,
-        c_ptr,
-        scales_ptr,
-        zeros_ptr,
-        g_ptr,
-        M,
-        N,
-        K,
-        bits,
-        maxq,
-        stride_am,
-        stride_ak,
-        stride_bk,
-        stride_bn,
-        stride_cm,
-        stride_cn,
-        stride_scales,
-        stride_zeros,
-        BLOCK_SIZE_M: tl.constexpr,
-        BLOCK_SIZE_N: tl.constexpr,
-        BLOCK_SIZE_K: tl.constexpr,
-        GROUP_SIZE_M: tl.constexpr,
-    ):
-        """
-        Compute the matrix multiplication C = A x B.
-        A is of shape (M, K) float16
-        B is of shape (K//8, N) int32
-        C is of shape (M, N) float16
-        scales is of shape (G, N) float16
-        zeros is of shape (G, N) float16
-        g_ptr is of shape (K) int32
-        """
-        infearure_per_bits = 32 // bits
-
-        pid = tl.program_id(axis=0)
-        num_pid_m = tl.cdiv(M, BLOCK_SIZE_M)
-        num_pid_n = tl.cdiv(N, BLOCK_SIZE_N)
-        num_pid_k = tl.cdiv(K, BLOCK_SIZE_K)
-        num_pid_in_group = GROUP_SIZE_M * num_pid_n
-        group_id = pid // num_pid_in_group
-        first_pid_m = group_id * GROUP_SIZE_M
-        group_size_m = min(num_pid_m - first_pid_m, GROUP_SIZE_M)
-        pid_m = first_pid_m + (pid % group_size_m)
-        pid_n = (pid % num_pid_in_group) // group_size_m
-
-        offs_am = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)
-        offs_bn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)
-        offs_k = tl.arange(0, BLOCK_SIZE_K)
-        a_ptrs = a_ptr + (
-            offs_am[:, None] * stride_am + offs_k[None, :] * stride_ak
-        )  # (BLOCK_SIZE_M, BLOCK_SIZE_K)
-        a_mask = offs_am[:, None] < M
-        # b_ptrs is set up such that it repeats elements along the K axis 8 times
-        b_ptrs = b_ptr + (
-            (offs_k[:, None] // infearure_per_bits) * stride_bk
-            + offs_bn[None, :] * stride_bn
-        )  # (BLOCK_SIZE_K, BLOCK_SIZE_N)
-        g_ptrs = g_ptr + offs_k
-        # shifter is used to extract the N bits of each element in the 32-bit word from B
-        scales_ptrs = scales_ptr + offs_bn[None, :]
-        zeros_ptrs = zeros_ptr + (offs_bn[None, :] // infearure_per_bits)
-
-        shifter = (offs_k % infearure_per_bits) * bits
-        zeros_shifter = (offs_bn % infearure_per_bits) * bits
-        accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32)
-
-        for k in range(0, num_pid_k):
-            g_idx = tl.load(g_ptrs)
-
-            # Fetch scales and zeros; these are per-outfeature and thus reused in the inner loop
-            scales = tl.load(
-                scales_ptrs + g_idx[:, None] * stride_scales
-            )  # (BLOCK_SIZE_K, BLOCK_SIZE_N,)
-            zeros = tl.load(
-                zeros_ptrs + g_idx[:, None] * stride_zeros
-            )  # (BLOCK_SIZE_K, BLOCK_SIZE_N,)
-
-            zeros = (zeros >> zeros_shifter[None, :]) & maxq
-            zeros = (zeros + 1) & maxq  # eventually avoid overflow
-
-            a = tl.load(a_ptrs, mask=a_mask, other=0.0)  # (BLOCK_SIZE_M, BLOCK_SIZE_K)
-            b = tl.load(b_ptrs)  # (BLOCK_SIZE_K, BLOCK_SIZE_N), but repeated
-
-            # Now we need to unpack b (which is N-bit values) into 32-bit values
-            b = (b >> shifter[:, None]) & maxq  # Extract the N-bit values
-            b = (b - zeros) * scales  # Scale and shift
-
-            accumulator += tl.dot(a, b)
-            a_ptrs += BLOCK_SIZE_K
-            b_ptrs += (BLOCK_SIZE_K // infearure_per_bits) * stride_bk
-            g_ptrs += BLOCK_SIZE_K
-
-        c_ptrs = c_ptr + stride_cm * offs_am[:, None] + stride_cn * offs_bn[None, :]
-        c_mask = (offs_am[:, None] < M) & (offs_bn[None, :] < N)
-        tl.store(c_ptrs, accumulator, mask=c_mask)
-
-except:
-    print("triton not installed.")
-
-
-def matmul248(input, qweight, scales, qzeros, g_idx, bits, maxq):
-    with torch.cuda.device(input.device):
-        output = torch.empty(
-            (input.shape[0], qweight.shape[1]), device=input.device, dtype=torch.float16
-        )
-        grid = lambda META: (
-            triton.cdiv(input.shape[0], META["BLOCK_SIZE_M"])
-            * triton.cdiv(qweight.shape[1], META["BLOCK_SIZE_N"]),
-        )
-        matmul_248_kernel[grid](
-            input,
-            qweight,
-            output,
-            scales,
-            qzeros,
-            g_idx,
-            input.shape[0],
-            qweight.shape[1],
-            input.shape[1],
-            bits,
-            maxq,
-            input.stride(0),
-            input.stride(1),
-            qweight.stride(0),
-            qweight.stride(1),
-            output.stride(0),
-            output.stride(1),
-            scales.stride(0),
-            qzeros.stride(0),
-        )
-        return output
-
-
-class QuantLinearFunction(torch.autograd.Function):
-    @staticmethod
-    @custom_fwd(cast_inputs=torch.float16)
-    def forward(ctx, input, qweight, scales, qzeros, g_idx, bits, maxq):
-        output = matmul248(input, qweight, scales, qzeros, g_idx, bits, maxq)
-        return output
-
-
-class QuantLinear(nn.Module):
-    def __init__(self, qweight, qzeros, scales, g_idx, bias, bits, groupsize):
-        super().__init__()
-        self.register_buffer("qweight", qweight)
-        self.register_buffer("qzeros", qzeros)
-        self.register_buffer("scales", scales)
-        self.register_buffer("g_idx", g_idx)
-        if bias is not None:
-            self.register_buffer("bias", bias)
-        else:
-            self.bias = None
-        if bits not in [2, 4, 8]:
-            raise NotImplementedError("Only 2,4,8 bits are supported.")
-        self.bits = bits
-        self.maxq = 2**self.bits - 1
-        self.groupsize = groupsize
-
-        self.outfeatures = qweight.shape[1]
-        self.infeatures = qweight.shape[0] * 32 // bits
-
-    @classmethod
-    def new(cls, bits, groupsize, infeatures, outfeatures, bias):
-        if bits not in [2, 4, 8]:
-            raise NotImplementedError("Only 2,4,8 bits are supported.")
-
-        qweight = torch.zeros((infeatures // 32 * bits, outfeatures), dtype=torch.int32)
-        qzeros = torch.zeros(
-            (math.ceil(infeatures / groupsize), outfeatures // 32 * bits),
-            dtype=torch.int32,
-        )
-        scales = torch.zeros(
-            (math.ceil(infeatures / groupsize), outfeatures), dtype=torch.float16
-        )
-        g_idx = torch.tensor(
-            [i // groupsize for i in range(infeatures)], dtype=torch.int32
-        )
-        if bias:
-            bias = torch.zeros((outfeatures), dtype=torch.float16)
-        else:
-            bias = None
-        return cls(qweight, qzeros, scales, g_idx, bias, bits, groupsize)
-
-    def pack(self, linear, scales, zeros, g_idx=None):
-        self.g_idx = g_idx.clone() if g_idx is not None else self.g_idx
-
-        scales = scales.t().contiguous()
-        zeros = zeros.t().contiguous()
-        scale_zeros = zeros * scales
-        self.scales = scales.clone().half()
-        if linear.bias is not None:
-            self.bias = linear.bias.clone().half()
-
-        intweight = []
-        for idx in range(self.infeatures):
-            intweight.append(
-                torch.round(
-                    (linear.weight.data[:, idx] + scale_zeros[self.g_idx[idx]])
-                    / self.scales[self.g_idx[idx]]
-                ).to(torch.int)[:, None]
-            )
-        intweight = torch.cat(intweight, dim=1)
-        intweight = intweight.t().contiguous()
-        intweight = intweight.numpy().astype(np.uint32)
-        qweight = np.zeros(
-            (intweight.shape[0] // 32 * self.bits, intweight.shape[1]), dtype=np.uint32
-        )
-        i = 0
-        row = 0
-        while row < qweight.shape[0]:
-            if self.bits in [2, 4, 8]:
-                for j in range(i, i + (32 // self.bits)):
-                    qweight[row] |= intweight[j] << (self.bits * (j - i))
-                i += 32 // self.bits
-                row += 1
-            else:
-                raise NotImplementedError("Only 2,4,8 bits are supported.")
-
-        qweight = qweight.astype(np.int32)
-        self.qweight = torch.from_numpy(qweight)
-
-        zeros -= 1
-        zeros = zeros.numpy().astype(np.uint32)
-        qzeros = np.zeros(
-            (zeros.shape[0], zeros.shape[1] // 32 * self.bits), dtype=np.uint32
-        )
-        i = 0
-        col = 0
-        while col < qzeros.shape[1]:
-            if self.bits in [2, 4, 8]:
-                for j in range(i, i + (32 // self.bits)):
-                    qzeros[:, col] |= zeros[:, j] << (self.bits * (j - i))
-                i += 32 // self.bits
-                col += 1
-            else:
-                raise NotImplementedError("Only 2,4,8 bits are supported.")
-
-        qzeros = qzeros.astype(np.int32)
-        self.qzeros = torch.from_numpy(qzeros)
-
-    def forward(self, x):
-        out_shape = x.shape[:-1] + (self.outfeatures,)
-        out = QuantLinearFunction.apply(
-            x.reshape(-1, x.shape[-1]),
-            self.qweight,
-            self.scales,
-            self.qzeros,
-            self.g_idx,
-            self.bits,
-            self.maxq,
-        )
-        out = out + self.bias if self.bias is not None else out
-        return out.reshape(out_shape)
diff --git a/server/text_generation_server/utils/hub.py b/server/text_generation_server/utils/hub.py
index b56484f6..f9c476ac 100644
--- a/server/text_generation_server/utils/hub.py
+++ b/server/text_generation_server/utils/hub.py
@@ -54,7 +54,6 @@ def _weight_files_from_dir(d: Path, extension: str) -> List[str]:
         if f.endswith(extension)
         and "arguments" not in f
         and "args" not in f
-        and "adapter" not in f
         and "training" not in f
     ]
     return filenames
diff --git a/server/text_generation_server/utils/import_utils.py b/server/text_generation_server/utils/import_utils.py
index 40e57646..782b4f15 100644
--- a/server/text_generation_server/utils/import_utils.py
+++ b/server/text_generation_server/utils/import_utils.py
@@ -1,13 +1,13 @@
 import torch
+from loguru import logger
+import os
 
 
-def is_xpu_available():
-    try:
-        import intel_extension_for_pytorch
-    except ImportError:
-        return False
+import importlib.util
 
-    return hasattr(torch, "xpu") and torch.xpu.is_available()
+
+def is_ipex_available():
+    return importlib.util.find_spec("intel_extension_for_pytorch") is not None
 
 
 def get_cuda_free_memory(device, memory_fraction):
@@ -18,11 +18,31 @@ def get_cuda_free_memory(device, memory_fraction):
 
 
 def get_xpu_free_memory(device, memory_fraction):
-    total_gpu_memory = torch.xpu.get_device_properties(device).total_memory
-    free_memory = int(total_gpu_memory * 0.5)
+    total_memory = torch.xpu.get_device_properties(device).total_memory
+    device_id = device.index
+    memory_fraction = float(os.getenv("XPU_MEMORY_FRACTION", "1.0"))
+    free_memory = max(
+        0,
+        int(
+            total_memory * 0.9 * memory_fraction - torch.xpu.memory_reserved(device_id)
+        ),
+    )
     return free_memory
 
 
+def get_cpu_free_memory(device, memory_fraction):
+    import psutil
+    from text_generation_server.utils.dist import WORLD_SIZE
+
+    mem = psutil.virtual_memory()
+    free_memory = int(mem.available * 0.95 / WORLD_SIZE)
+    return free_memory
+
+
+def noop(*args, **kwargs):
+    pass
+
+
 SYSTEM = None
 if torch.version.hip is not None:
     SYSTEM = "rocm"
@@ -34,17 +54,22 @@ elif torch.version.cuda is not None and torch.cuda.is_available():
     empty_cache = torch.cuda.empty_cache
     synchronize = torch.cuda.synchronize
     get_free_memory = get_cuda_free_memory
-elif is_xpu_available():
-    SYSTEM = "xpu"
-    empty_cache = torch.xpu.empty_cache
-    synchronize = torch.xpu.synchronize
-    get_free_memory = get_xpu_free_memory
+elif is_ipex_available():
+    SYSTEM = "ipex"
+    import intel_extension_for_pytorch  # noqa: F401
+
+    if hasattr(torch, "xpu") and torch.xpu.is_available():
+        empty_cache = torch.xpu.empty_cache
+        synchronize = torch.xpu.synchronize
+        get_free_memory = get_xpu_free_memory
+    else:
+        empty_cache = noop
+        synchronize = noop
+        get_free_memory = get_cpu_free_memory
 else:
     SYSTEM = "cpu"
 
-    def noop(*args, **kwargs):
-        pass
-
     empty_cache = noop
     synchronize = noop
-    get_free_memory = noop
+    get_free_memory = get_cpu_free_memory
+logger.info(f"Detected system {SYSTEM}")
diff --git a/server/text_generation_server/utils/log.py b/server/text_generation_server/utils/log.py
index b1456f1e..4385c71e 100644
--- a/server/text_generation_server/utils/log.py
+++ b/server/text_generation_server/utils/log.py
@@ -1,6 +1,15 @@
 from functools import lru_cache
+from text_generation_server.utils.dist import RANK
 
 
 @lru_cache(10)
-def log_once(log, msg: str):
-    log(msg)
+def log_once(log, msg: str, master=True):
+    if master:
+        log_master(log, msg)
+    else:
+        log(msg)
+
+
+def log_master(log, msg: str):
+    if RANK == 0:
+        log(msg)
diff --git a/server/text_generation_server/utils/merges/strategies.py b/server/text_generation_server/utils/merges/strategies.py
new file mode 100644
index 00000000..cb39cde1
--- /dev/null
+++ b/server/text_generation_server/utils/merges/strategies.py
@@ -0,0 +1,220 @@
+import copy
+from abc import ABC
+from collections import defaultdict
+from typing import TYPE_CHECKING, Dict, List, Tuple, Type, Union
+from text_generation_server.utils.merges.utils import (
+    calculate_majority_sign_mask,
+    disjoint_merge,
+    prune,
+)
+import torch
+
+if TYPE_CHECKING:
+    from text_generation_server.adapters.lora import LoraConfig
+    from text_generation_server.utils.adapter import ModuleMap
+
+
+class AdapterParameters:
+    def __init__(
+        self, adapter_ids, weights, merge_strategy, density, majority_sign_method
+    ):
+        self.adapter_ids = adapter_ids
+        self.weights = weights
+        self.merge_strategy = merge_strategy
+        self.density = density
+        self.majority_sign_method = majority_sign_method
+
+
+def _apply_weights(
+    tensors: Union[torch.Tensor, List[torch.Tensor]], w: torch.Tensor
+) -> torch.Tensor:
+    if isinstance(tensors, torch.Tensor):
+        t = tensors
+    else:
+        t = torch.stack(tensors, dim=0)
+
+    # element-wise weighting of each task tensor
+    # need to unsqueeze weights to match task tensor dimensions
+    # for multiplication to apply element-wise
+    while len(t.shape) > len(w.shape):
+        w = w.unsqueeze(-1)
+    return t * w
+
+
+class MergeStrategy(ABC):
+    def merge(
+        self, task_tensors: List[torch.Tensor], weights: torch.Tensor
+    ) -> torch.Tensor:
+        raise NotImplementedError()
+
+
+class LinearMerge(MergeStrategy):
+    def __init__(self, **kwargs):
+        pass
+
+    def merge(
+        self, task_tensors: List[torch.Tensor], weights: torch.Tensor
+    ) -> torch.Tensor:
+        weighted_task_tensors = _apply_weights(task_tensors, weights)
+        return weighted_task_tensors.sum(dim=0)
+
+
+class TiesMerge(MergeStrategy):
+    def __init__(self, density: float, majority_sign_method: str = "total", **kwargs):
+        self.density = density
+        self.majority_sign_method = majority_sign_method
+
+    def merge(
+        self, task_tensors: List[torch.Tensor], weights: torch.Tensor
+    ) -> torch.Tensor:
+        # sparsify
+        task_tensors = [
+            prune(tensor, self.density, method="magnitude") for tensor in task_tensors
+        ]
+        task_tensors = torch.stack(task_tensors, dim=0)
+
+        # elect sign before applying weights
+        majority_sign_mask = calculate_majority_sign_mask(
+            task_tensors, method=self.majority_sign_method
+        )
+        weighted_task_tensors = _apply_weights(task_tensors, weights)
+
+        # disjoint merge
+        return disjoint_merge(weighted_task_tensors, majority_sign_mask)
+
+
+class DareLinearMerge(MergeStrategy):
+    def __init__(self, density: float, **kwargs):
+        self.density = density
+
+    def merge(
+        self, task_tensors: List[torch.Tensor], weights: torch.Tensor
+    ) -> torch.Tensor:
+        # sparsify
+        task_tensors = [
+            prune(tensor, self.density, method="random", rescale=True)
+            for tensor in task_tensors
+        ]
+        weighted_task_tensors = _apply_weights(task_tensors, weights)
+        return weighted_task_tensors.sum(dim=0)
+
+
+class DareTiesMerge(MergeStrategy):
+    def __init__(self, density: float, majority_sign_method: str = "total", **kwargs):
+        self.density = density
+        self.majority_sign_method = majority_sign_method
+
+    def merge(
+        self, task_tensors: List[torch.Tensor], weights: torch.Tensor
+    ) -> torch.Tensor:
+        # sparsify
+        task_tensors = [
+            prune(tensor, self.density, method="random", rescale=True)
+            for tensor in task_tensors
+        ]
+        task_tensors = torch.stack(task_tensors, dim=0)
+
+        # elect sign before applying weights
+        majority_sign_mask = calculate_majority_sign_mask(
+            task_tensors, method=self.majority_sign_method
+        )
+        weighted_task_tensors = _apply_weights(task_tensors, weights)
+
+        # disjoint merge
+        mixed_task_tensors = disjoint_merge(weighted_task_tensors, majority_sign_mask)
+        return mixed_task_tensors
+
+
+strategy_registry: Dict[str, Type[MergeStrategy]] = {
+    "linear": LinearMerge,
+    "ties": TiesMerge,
+    "dare_linear": DareLinearMerge,
+    "dare_ties": DareTiesMerge,
+}
+
+
+def merge_adapters(
+    adapters: List[Tuple["ModuleMap", "LoraConfig"]],
+    merge_params: AdapterParameters,
+) -> Tuple["ModuleMap", "LoraConfig"]:
+    # strategy_name = MergeStrategyEnum.Name(merge_params.merge_strategy).lower()
+    strategy_name = "linear"
+
+    weights = merge_params.weights
+    if not weights:
+        weights = torch.ones(len(adapters))
+    else:
+        weights = torch.tensor(weights)
+
+    merge_config = {
+        "density": merge_params.density,
+        # "majority_sign_method": MajoritySignMethodEnum.Name(
+        #     merge_params.majority_sign_method
+        # ).lower(),
+        "majority_sign_method": "total",
+    }
+    merge_strategy = strategy_registry[strategy_name](**merge_config)
+
+    module_maps: Dict[str, Dict[str, Dict[str, List[torch.Tensor]]]] = defaultdict(
+        lambda: defaultdict(lambda: defaultdict(list))
+    )
+    lora_configs = []
+    weight_name_to_adapter_idx = defaultdict(list)
+
+    # input is list of (module_map, lora_config) tuples
+    # convert into dict[k][param_name] -> list of tensors
+    for idx, (module_map, lora_config) in enumerate(adapters):
+        for weight_name, data in module_map.items():
+            weight_name_to_adapter_idx[weight_name].append(idx)
+            for k, (param_data, param_name) in data.items():
+                module_maps[weight_name][k][param_name].append(param_data)
+        lora_configs.append(lora_config)
+
+    # validate lora configs are compatible
+    _validate_lora_configs(lora_configs)
+
+    # merge tensors for each module such that we have a single ModuleMap:
+    # dict[k] -> merged tensor
+    merged_module_map: "ModuleMap" = defaultdict(dict)
+    for weight_name, data in module_maps.items():
+        indices = weight_name_to_adapter_idx[weight_name]
+        param_weights = weights[indices]
+        for k, param_data in data.items():
+            for param_name, tensors in param_data.items():
+                merged_tensor = merge_strategy.merge(tensors, param_weights)
+                merged_module_map[weight_name][k] = (merged_tensor, param_name)
+
+    # merge lora configs
+    merged_lora_config = _merge_lora_configs(lora_configs)
+
+    return merged_module_map, merged_lora_config
+
+
+def _validate_lora_configs(lora_configs: List["LoraConfig"]):
+    # check that all configs have the same rank
+    ranks = set(lora_config.r for lora_config in lora_configs)
+    if len(ranks) > 1:
+        raise ValueError(
+            f"unable to merge adapters, lora configs have different ranks: {ranks}"
+        )
+
+    if all(len(lora_config.target_modules) == 0 for lora_config in lora_configs):
+        raise ValueError(
+            "unable to merge adapters, lora configs have no target modules"
+        )
+
+
+def _merge_lora_configs(lora_configs: List["LoraConfig"]) -> "LoraConfig":
+    merged_lora_config = copy.copy(lora_configs[0])
+
+    # merge target modules as a union operation
+    merged_target_modules = sorted(
+        set(
+            module
+            for lora_config in lora_configs
+            for module in lora_config.target_modules
+        )
+    )
+    merged_lora_config.target_modules = merged_target_modules
+
+    return merged_lora_config
diff --git a/server/text_generation_server/utils/merges/utils.py b/server/text_generation_server/utils/merges/utils.py
new file mode 100644
index 00000000..d9ad3278
--- /dev/null
+++ b/server/text_generation_server/utils/merges/utils.py
@@ -0,0 +1,108 @@
+# coding=utf-8
+# From: https://github.com/huggingface/peft/pull/1364
+# Copyright 2024-present the HuggingFace Inc. team.
+# Modifications by Predibase, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Literal
+
+import torch
+
+
+def magnitude_based_pruning(tensor: torch.Tensor, density: float) -> torch.Tensor:
+    """
+    Prune the smallest values of the task tensors and retain the top-k values based on the specified fraction
+    `density`.
+
+    Args:
+    tensor (`torch.Tensor`):The tensor to prune.
+    density (`float`):The fraction of values to preserve. Should be in [0,1].
+    """
+    mask = torch.zeros_like(tensor).reshape(-1)
+    k = int(density * tensor.reshape(-1).shape[0])
+    top_k = torch.topk(tensor.abs().reshape(-1), k=k, largest=True)
+    mask[top_k[1]] = 1
+    return tensor * mask.reshape(tensor.shape)
+
+
+def random_pruning(tensor: torch.Tensor, density: float, rescale: bool) -> torch.Tensor:
+    """
+    Prune the smallest values of the task tensors and retain the top-k values based on the specified fraction
+    `density`.
+
+    Args:
+    tensor (`torch.Tensor`):The tensor to prune.
+    density (`float`):The fraction of values to preserve. Should be in [0,1].
+    rescale (`bool`):Whether to rescale the result to preserve the expected value of the original tensor.
+    """
+    mask = torch.bernoulli(torch.full_like(input=tensor, fill_value=density))
+    pruned_tensor = tensor * mask
+    if rescale:
+        torch.div(input=pruned_tensor, other=density)
+    return pruned_tensor
+
+
+def prune(
+    tensor: torch.Tensor,
+    density: float,
+    method: Literal["magnitude", "random"],
+    rescale: bool = False,
+) -> torch.Tensor:
+    """
+    Prune the values of task tensors based on the `method`.
+
+    Args:
+    tensor (`torch.Tensor`):The tensor to prune.
+    density (`float`):The fraction of values to preserve. Should be in [0,1].
+    method (`str`):The method to use to prune. Should be one of ["magnitude", "random"].
+    rescale (`bool`):Whether to rescale the result to preserve the expected value of the original tensor.
+    """
+    if density >= 1:
+        return tensor
+    elif density < 0:
+        raise ValueError("Density should be >= 0, got {density}")
+    if method == "magnitude":
+        return magnitude_based_pruning(tensor, density)
+    elif method == "random":
+        return random_pruning(tensor, density, rescale=rescale)
+    else:
+        raise ValueError(f"Unknown method {method}")
+
+
+def calculate_majority_sign_mask(
+    tensor: torch.Tensor, method: Literal["total", "frequency"] = "total"
+):
+    """
+    Get the mask of the majority sign across the task tensors. Task tensors are stacked on dimension 0.
+
+    Args:
+    tensor (`torch.Tensor`):The tensor to get the mask from.
+    method (`str`):The method to use to get the mask. Should be one of ["total", "frequency"].
+    """
+
+    sign = tensor.sign()
+    if method == "total":
+        sign_magnitude = (sign * tensor.abs()).sum(dim=0)
+    elif method == "frequency":
+        sign_magnitude = sign.sum(dim=0)
+    else:
+        raise RuntimeError(f'Unimplemented mask method "{method}"')
+    majority_sign = torch.where(sign_magnitude >= 0, 1, -1)
+    return sign == majority_sign
+
+
+def disjoint_merge(task_tensors, majority_sign_mask):
+    mixed_task_tensors = (task_tensors * majority_sign_mask).sum(dim=0)
+    num_params_preserved = majority_sign_mask.sum(dim=0)
+    return mixed_task_tensors / torch.clamp(num_params_preserved, min=1.0)
diff --git a/server/text_generation_server/utils/paged_attention.py b/server/text_generation_server/utils/paged_attention.py
deleted file mode 100644
index 6cc30e6d..00000000
--- a/server/text_generation_server/utils/paged_attention.py
+++ /dev/null
@@ -1,137 +0,0 @@
-import torch
-from text_generation_server.utils.import_utils import SYSTEM
-
-_PARTITION_SIZE = 512
-
-if SYSTEM == "xpu":
-    import intel_extension_for_pytorch as ipex
-else:
-    try:
-        from vllm._C import cache_ops
-        from vllm._C import ops
-    except Exception as e:
-        raise ImportError(
-            f"Could not import vllm paged attention. Make sure your installation is correct. Complete error: {e}"
-        )
-
-
-def reshape_and_cache(
-    key: torch.Tensor,
-    value: torch.Tensor,
-    key_cache: torch.Tensor,
-    value_cache: torch.Tensor,
-    slots: torch.Tensor,
-):
-    if SYSTEM == "xpu":
-        ipex.llm.modules.PagedAttention.reshape_and_cache(
-            key, value, key_cache, value_cache, slots
-        )
-    else:
-        cache_ops.reshape_and_cache(
-            key, value, key_cache, value_cache, slots, "auto", 1.0
-        )
-
-
-def attention(
-    out: torch.Tensor,
-    query: torch.Tensor,
-    key_cache: torch.Tensor,
-    value_cache: torch.Tensor,
-    kv_head_mapping: torch.Tensor,
-    softmax_scale: float,
-    block_tables: torch.Tensor,
-    input_lengths: torch.Tensor,
-    max_s: int,
-):
-    # Adapted from: https://github.com/vllm-project/vllm/blob/f8a1e39fae05ca610be8d5a78be9d40f5274e5fc/vllm/model_executor/layers/attention.py
-    # Copyright 2023 The vLLM team. All rights
-    # reserved.
-    #
-    # Licensed under the Apache License, Version 2.0 (the "License");
-    # you may not use this file except in compliance with the License.
-    # You may obtain a copy of the License at
-    #
-    #     http://www.apache.org/licenses/LICENSE-2.0
-    #
-    # Unless required by applicable law or agreed to in writing, software
-    # distributed under the License is distributed on an "AS IS" BASIS,
-    # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    # See the License for the specific language governing permissions and
-    # limitations under the License.
-    #
-
-    # value_cache => [num_blocks, num_heads, head_size, block_size]
-    block_size = value_cache.shape[3]
-    num_seqs, num_heads, head_size = query.shape
-    max_num_partitions = (max_s + _PARTITION_SIZE - 1) // _PARTITION_SIZE
-    if SYSTEM == "xpu":
-        query = query.contiguous()
-        return ipex.llm.modules.PagedAttention.single_query_cached_kv_attention(
-            out,
-            query,
-            key_cache,
-            value_cache,
-            kv_head_mapping,
-            softmax_scale,
-            block_tables,
-            input_lengths,
-            block_size,
-            max_s,
-            None,
-        )
-
-    # NOTE(woosuk): We use a simple heuristic to decide whether to use
-    # PagedAttention V1 or V2. If the number of partitions is 1, we use
-    # V1 to avoid the overhead of reduction. Also, if the number of
-    # sequences or heads is large, we use V1 since there is enough work
-    # to parallelize.
-    use_v1 = max_s <= 8192 and (max_num_partitions == 1 or num_seqs * num_heads > 512)
-    if use_v1:
-        ops.paged_attention_v1(
-            out,
-            query,
-            key_cache,
-            value_cache,
-            kv_head_mapping,
-            softmax_scale,
-            block_tables,
-            input_lengths,
-            block_size,
-            max_s,
-            None,
-            "auto",
-            1.0,
-        )
-    else:
-        # Run PagedAttention V2.
-        assert _PARTITION_SIZE % block_size == 0
-        tmp_output = torch.empty(
-            size=(num_seqs, num_heads, max_num_partitions, head_size),
-            dtype=out.dtype,
-            device=out.device,
-        )
-        exp_sums = torch.empty(
-            size=(num_seqs, num_heads, max_num_partitions),
-            dtype=torch.float32,
-            device=out.device,
-        )
-        max_logits = torch.empty_like(exp_sums)
-
-        ops.paged_attention_v2(
-            out,
-            exp_sums,
-            max_logits,
-            tmp_output,
-            query,
-            key_cache,
-            value_cache,
-            kv_head_mapping,
-            softmax_scale,
-            block_tables,
-            input_lengths,
-            block_size,
-            max_s,
-            None,
-            "auto",
-            1.0,
-        )
diff --git a/server/text_generation_server/utils/peft.py b/server/text_generation_server/utils/peft.py
index 48ca264b..d49e73f0 100644
--- a/server/text_generation_server/utils/peft.py
+++ b/server/text_generation_server/utils/peft.py
@@ -1,5 +1,5 @@
 import os
-import json
+from typing import Union
 from loguru import logger
 import torch
 
@@ -28,7 +28,7 @@ def download_and_unload_peft(model_id, revision, trust_remote_code):
             low_cpu_mem_usage=True,
         )
     logger.info("Peft model detected.")
-    logger.info(f"Merging the lora weights.")
+    logger.info("Merging the lora weights.")
 
     base_model_id = model.peft_config["default"].base_model_name_or_path
 
@@ -43,3 +43,26 @@ def download_and_unload_peft(model_id, revision, trust_remote_code):
     model.save_pretrained(cache_dir, safe_serialization=True)
     model.config.save_pretrained(cache_dir)
     tokenizer.save_pretrained(cache_dir)
+
+
+def download_peft(
+    model_id: Union[str, os.PathLike], revision: str, trust_remote_code: bool
+):
+    torch_dtype = torch.float16
+    try:
+        _model = AutoPeftModelForCausalLM.from_pretrained(
+            model_id,
+            revision=revision,
+            torch_dtype=torch_dtype,
+            trust_remote_code=trust_remote_code,
+            low_cpu_mem_usage=True,
+        )
+    except Exception:
+        _model = AutoPeftModelForSeq2SeqLM.from_pretrained(
+            model_id,
+            revision=revision,
+            torch_dtype=torch_dtype,
+            trust_remote_code=trust_remote_code,
+            low_cpu_mem_usage=True,
+        )
+    logger.info("Peft model downloaded.")
diff --git a/server/text_generation_server/utils/quantization.py b/server/text_generation_server/utils/quantization.py
new file mode 100644
index 00000000..ee561acc
--- /dev/null
+++ b/server/text_generation_server/utils/quantization.py
@@ -0,0 +1,202 @@
+import json
+import os
+from dataclasses import dataclass
+from typing import Optional
+
+from huggingface_hub import hf_hub_download
+from text_generation_server.layers.marlin.gptq import can_use_gptq_marlin
+from text_generation_server.utils.weights import (
+    DefaultWeightsLoader,
+    WeightsLoader,
+)
+
+
+# TODO: Split this config to have a single config type per quant method
+@dataclass
+class _QuantizerConfig:
+    bits: int
+    checkpoint_format: Optional[str]
+    desc_act: bool
+    groupsize: int
+    quant_method: str
+    sym: bool
+
+
+@dataclass
+class _FP8QuantizerConfig:
+    activation_scale_ub: float
+
+
+# We should probably do this with Pytantic JSON deserialization,
+# but for now we'll stay close to the old _set_gptq_params.
+def _get_quantizer_config(model_id, revision):
+    bits = 4
+    groupsize = -1
+    quant_method = "gptq"
+    checkpoint_format = None
+    sym = False
+    desc_act = False
+
+    filename = "config.json"
+    try:
+        if os.path.exists(os.path.join(model_id, filename)):
+            filename = os.path.join(model_id, filename)
+        else:
+            filename = hf_hub_download(model_id, filename=filename, revision=revision)
+        with open(filename, "r") as f:
+            data = json.load(f)
+
+        # FP8 config
+        if data["quantization_config"]["quant_method"] == "fbgemm_fp8":
+            return _FP8QuantizerConfig(
+                activation_scale_ub=data["quantization_config"]["activation_scale_ub"]
+            )
+
+        if "zero_point" in data["quantization_config"]:
+            sym = not data["quantization_config"]["zero_point"]
+            quant_method = "awq"
+        elif "sym" in data["quantization_config"]:
+            sym = data["quantization_config"]["sym"]
+
+        bits = data["quantization_config"]["bits"]
+        groupsize = data["quantization_config"]["group_size"]
+        # Order is important here, desc_act is missing on some real models
+        quant_method = data["quantization_config"]["quant_method"]
+        checkpoint_format = data["quantization_config"].get("checkpoint_format")
+        desc_act = data["quantization_config"]["desc_act"]
+    except Exception:
+        filename = "quantize_config.json"
+        try:
+            if os.path.exists(os.path.join(model_id, filename)):
+                filename = os.path.join(model_id, filename)
+            else:
+                filename = hf_hub_download(
+                    model_id, filename=filename, revision=revision
+                )
+            with open(filename, "r") as f:
+                data = json.load(f)
+            bits = data["bits"]
+            groupsize = data["group_size"]
+
+            if "zero_point" in data:
+                sym = not data["zero_point"]
+                quant_method = "awq"
+            elif "sym" in data:
+                sym = data["sym"]
+
+            desc_act = data["desc_act"]
+            if "version" in data and data["version"] == "GEMM":
+                quant_method = "awq"
+        except Exception:
+            filename = "quant_config.json"
+            try:
+                if os.path.exists(os.path.join(model_id, filename)):
+                    filename = os.path.join(model_id, filename)
+                else:
+                    filename = hf_hub_download(
+                        model_id, filename=filename, revision=revision
+                    )
+                with open(filename, "r") as f:
+                    data = json.load(f)
+                bits = data["w_bit"]
+                groupsize = data["q_group_size"]
+                desc_act = data["desc_act"]
+                if "version" in data and data["version"] == "GEMM":
+                    quant_method = "awq"
+            except Exception:
+                pass
+
+    return _QuantizerConfig(
+        bits=bits,
+        groupsize=groupsize,
+        quant_method=quant_method,
+        checkpoint_format=checkpoint_format,
+        sym=sym,
+        desc_act=desc_act,
+    )
+
+
+def get_loader(
+    quantize: Optional[str], model_id: str, revision: Optional[str]
+) -> WeightsLoader:
+    quantizer_config = _get_quantizer_config(model_id, revision)
+    if quantize in {"awq", "gptq"}:
+        from text_generation_server.layers.gptq import GPTQWeightsLoader
+
+        # TODO: improve check once we have one config type per quantize value
+        if not isinstance(quantizer_config, _QuantizerConfig):
+            raise ValueError(
+                f"Quantize is set to `{quantize}` but received a `{quantizer_config.__class__.__name__}` config."
+            )
+
+        if can_use_gptq_marlin(
+            bits=quantizer_config.bits,
+            groupsize=quantizer_config.groupsize,
+            quant_method=quantizer_config.quant_method,
+            quantize=quantize,
+            sym=quantizer_config.sym,
+        ):
+            from text_generation_server.layers.marlin import GPTQMarlinWeightsLoader
+
+            return GPTQMarlinWeightsLoader(
+                bits=quantizer_config.bits,
+                desc_act=quantizer_config.desc_act,
+                groupsize=quantizer_config.groupsize,
+                quant_method=quantizer_config.quant_method,
+                quantize=quantize,
+                sym=quantizer_config.sym,
+            )
+        else:
+            return GPTQWeightsLoader(
+                bits=quantizer_config.bits,
+                desc_act=quantizer_config.desc_act,
+                groupsize=quantizer_config.groupsize,
+                quant_method=quantizer_config.quant_method,
+                quantize=quantize,
+                sym=quantizer_config.sym,
+            )
+    elif quantize == "bitsandbytes":
+        from text_generation_server.layers.bnb import BNBWeight
+
+        return DefaultWeightsLoader(BNBWeight)
+    elif quantize == "bitsandbytes-fp4":
+        from text_generation_server.layers.bnb import BNBFP4Weight
+
+        return DefaultWeightsLoader(BNBFP4Weight)
+    elif quantize == "bitsandbytes-nf4":
+        from text_generation_server.layers.bnb import BNBNF4Weight
+
+        return DefaultWeightsLoader(BNBNF4Weight)
+    elif quantize == "eetq":
+        from text_generation_server.layers.eetq import EETQWeight
+
+        return DefaultWeightsLoader(EETQWeight)
+    elif quantize == "exl2":
+        from text_generation_server.layers.exl2 import Exl2WeightsLoader
+
+        return Exl2WeightsLoader()
+    elif quantize == "marlin":
+        from text_generation_server.layers.marlin import MarlinWeightsLoader
+
+        # TODO: improve check once we have one config type per quantize value
+        if not isinstance(quantizer_config, _QuantizerConfig):
+            raise ValueError(
+                f"Quantize is set to `{quantize}` but received a `{quantizer_config.__class__.__name__}` config."
+            )
+
+        return MarlinWeightsLoader(
+            bits=quantizer_config.bits,
+            is_marlin_24=quantizer_config.checkpoint_format == "marlin_24",
+        )
+    elif quantize == "fp8" or quantize is None:
+        from text_generation_server.layers.fp8 import HybridFP8UnquantLoader
+
+        # Since the default for the quantize config is _QuantizerConfig,
+        # we need to add this check to not get an attribute error
+        activation_scale_ub = None
+        if isinstance(quantizer_config, _FP8QuantizerConfig):
+            activation_scale_ub = quantizer_config.activation_scale_ub
+
+        return HybridFP8UnquantLoader(activation_scale_ub, to_fp8=quantize == "fp8")
+    else:
+        raise ValueError(f"Unknown quantization method: {quantize}")
diff --git a/server/text_generation_server/utils/segments.py b/server/text_generation_server/utils/segments.py
new file mode 100644
index 00000000..f5961102
--- /dev/null
+++ b/server/text_generation_server/utils/segments.py
@@ -0,0 +1,66 @@
+# Origin:   https://github.com/predibase/lorax
+# Path:     lorax/server/lorax_server/utils/segments.py
+# License:  Apache License Version 2.0, January 2004
+
+from typing import List, Tuple, Union
+
+import torch
+
+
+def find_segments(
+    adapter_indices: Union[torch.Tensor, List[int]]
+) -> Tuple[List[int], List[int]]:
+    segments = [0]
+    segment_indices = []
+
+    if isinstance(adapter_indices, torch.Tensor):
+        # Calling .item() repeatedly on CUDA tensor is very slow, so we move it to CPU first
+        adapter_indices = adapter_indices.cpu().tolist()
+
+    start_index = 0
+    for i in range(1, len(adapter_indices)):
+        if adapter_indices[i] != adapter_indices[i - 1]:
+            segments.append(i)
+            segment_indices.append(adapter_indices[i - 1])
+            start_index = i
+
+    # Handle the last segment
+    if start_index < len(adapter_indices):
+        segments.append(len(adapter_indices))
+        segment_indices.append(adapter_indices[-1])
+
+    return segments, segment_indices
+
+
+class SegmentConcatBuilder:
+    def __init__(self):
+        self.adapter_segment_indices = []
+        self.adapter_segment_tensors = []
+
+    def concat(self, adapter_segments: torch.Tensor, segment_indices: List[int]):
+        # Update adapter segments
+        if self.adapter_segment_tensors:
+            # Because we have already processed at least one batch, remove the 0 start index
+            # from this batch denoting the beginning of the segment, then offset all segment
+            # positions by the value of the last segment in the previous batch to account for
+            # the concatenation.
+            adapter_segments = (
+                adapter_segments[1:] + self.adapter_segment_tensors[-1][-1]
+            )
+
+        if (
+            self.adapter_segment_indices
+            and self.adapter_segment_indices[-1] == segment_indices[0]
+        ):
+            # If the last segment in the previous batch is the same as the first segment in this batch,
+            # then we merge them together into a single segment. In effect, this means removing it from
+            # the segment indices of this batch, and extending the segment span by removing the segment
+            # end index from the previous batch.
+            segment_indices = segment_indices[1:]
+            self.adapter_segment_tensors[-1] = self.adapter_segment_tensors[-1][:-1]
+
+        self.adapter_segment_indices.extend(segment_indices)
+        self.adapter_segment_tensors.append(adapter_segments)
+
+    def build(self) -> Tuple[torch.Tensor, List[int]]:
+        return torch.concat(self.adapter_segment_tensors), self.adapter_segment_indices
diff --git a/server/text_generation_server/utils/sgmv.py b/server/text_generation_server/utils/sgmv.py
new file mode 100644
index 00000000..2d0a73a5
--- /dev/null
+++ b/server/text_generation_server/utils/sgmv.py
@@ -0,0 +1,252 @@
+# Origin:   https://github.com/predibase/lorax
+# Path:     lorax/server/lorax_server/utils/sgmv.py
+# License:  Apache License Version 2.0, January 2004
+
+import os
+import warnings
+from functools import lru_cache
+from typing import List, Tuple
+
+import torch
+import torch.nn.functional as F
+
+try:
+    import punica_kernels as _kernels
+
+    HAS_SGMV = not bool(os.environ.get("DISABLE_SGMV", ""))
+except ImportError:
+    warnings.warn("Could not import SGMV kernel from Punica, falling back to loop.")
+    _kernels = None
+    HAS_SGMV = False
+
+
+MIN_SGMV_RANK = 8
+MIN_RANK_CUSTOM = 16
+MAX_RANK_CUSTOM = 128
+SGMV_BLOCK_SIZE = 16
+BGMV_MAX_RANK = 64
+
+
+def has_sgmv() -> bool:
+    return HAS_SGMV
+
+
+def pad_rank(t: torch.Tensor, dim: int, world_size: int) -> torch.Tensor:
+    """Pad a tensor to the minimum rank for SGMV and the nearest multiple of the SGMV block size."""
+    if not has_sgmv():
+        return t
+
+    # tensor parallelism will result in effective rank being divided by world_size,
+    # so we need to scale the min rank to offset that effect
+    min_rank = MIN_SGMV_RANK * world_size
+
+    # if we're at or below the min rank, pad up to the min rank
+    # otherwise, pad to the nearest multiple of the block size
+    current_rank = t.size(dim)
+    target_rank = (
+        min_rank
+        if current_rank <= min_rank
+        else (current_rank + SGMV_BLOCK_SIZE - 1) // SGMV_BLOCK_SIZE * SGMV_BLOCK_SIZE
+    )
+    if current_rank == target_rank:
+        return t
+
+    pad_size = target_rank - current_rank
+
+    # see complicatd pad syntax here: https://pytorch.org/docs/stable/generated/torch.nn.functional.pad.html
+    pad = [0, 0] * t.dim()
+    pad[(t.dim() - dim - 1) * 2 + 1] = pad_size
+    pad = tuple(pad)
+
+    return F.pad(t, pad, mode="constant", value=0.0)
+
+
+def use_cutlass_shrink(lora_rank: int) -> bool:
+    return lora_rank < MIN_RANK_CUSTOM
+
+
+def orient_for_rank(t: torch.Tensor, rank: int) -> torch.Tensor:
+    if MIN_RANK_CUSTOM <= rank <= MAX_RANK_CUSTOM:
+        return t.transpose(0, 1)
+    return t
+
+
+# Source: https://github.com/punica-ai/punica/blob/master/src/punica/ops/__init__.py
+def add_lora_sgmv_cutlass(
+    y: torch.Tensor,
+    x: torch.Tensor,
+    wa_ptr: torch.Tensor,
+    wb_ptr: torch.Tensor,
+    s_start: torch.Tensor,
+    s_end: torch.Tensor,
+    layer_idx: int,
+    lora_rank: int,
+):
+    """
+    Semantics:
+        y[s[i]:s[i+1]] += x[s[i]:s[i+1]] @ deref(wa_ptr[i]).T @ deref(wb_ptr[i])
+
+    Args:
+        y: Shape: `[B, H2]`. Output vectors. Will be changed in-place.
+        x: Shape: `[B, H1]`. Input vectors.
+        wa_ptr: Shape: `[S]`. DType: torch.int64. Pointer to the weight matrices.\
+            Weight matrix shape: `[num_layers, R, H1]`.
+        wb_ptr: Shape: `[S]`. DType: torch.int64. Pointer to the weight matrices.\
+            Weight matrix shape: `[num_layers, R, H2]`.
+        s_start: Shape: `[S]`, DType: torch.int32. Indptr of the weight matrices start indices.
+        s_end: Shape: `[S]`, DType: torch.int32. Indptr of the weight matrices end indices.
+        layer_idx: Layer index of the weight matrices.
+    """
+    if lora_rank < MIN_RANK_CUSTOM or lora_rank > MAX_RANK_CUSTOM:
+        # Custom SGMV shrink only supports rank 16, 32, 64, 128
+        _add_lora_sgmv_cutlass_legacy(
+            y, x, wa_ptr, wb_ptr, s_start, s_end, layer_idx, lora_rank
+        )
+        return
+
+    tmp1 = torch.empty((8 * 1024 * 1024,), dtype=torch.uint8, device=x.device)
+    tmp2_size = _kernels.sgmv_cutlass_tmp_size(wa_ptr.size(0))
+    tmp2 = torch.empty((tmp2_size,), dtype=torch.uint8, device=x.device)
+    v = torch.zeros((x.size(0), lora_rank), dtype=x.dtype, device=x.device)
+    _kernels.sgmv_shrink(v, x, wa_ptr, s_start, s_end, tmp1, layer_idx)
+    _kernels.sgmv_cutlass(y, v, wb_ptr, s_start, s_end, tmp2, layer_idx)
+
+
+def _add_lora_sgmv_cutlass_legacy(
+    y: torch.Tensor,
+    x: torch.Tensor,
+    wa_ptr: torch.Tensor,
+    wb_ptr: torch.Tensor,
+    s_start: torch.IntTensor,
+    s_end: torch.IntTensor,
+    layer_idx: int,
+    lora_rank: int,
+):
+    tmp_size = _kernels.sgmv_cutlass_tmp_size(wa_ptr.size(0))
+    tmp = torch.empty((tmp_size,), dtype=torch.uint8, device=x.device)
+    v = torch.zeros((x.size(0), lora_rank), dtype=x.dtype, device=x.device)
+    _kernels.sgmv_cutlass(v, x, wa_ptr, s_start, s_end, tmp, layer_idx)
+    _kernels.sgmv_cutlass(y, v, wb_ptr, s_start, s_end, tmp, layer_idx)
+
+
+@lru_cache(maxsize=1)
+def get_tmp_tensor(device: torch.device) -> torch.Tensor:
+    return torch.empty((8 * 1024 * 1024,), dtype=torch.uint8, device=device)
+
+
+@lru_cache(maxsize=32)
+def get_tmp_tensor_for_size(size: int, device: torch.device) -> torch.Tensor:
+    tmp_size = _kernels.sgmv_cutlass_tmp_size(size)
+    return torch.empty((tmp_size,), dtype=torch.uint8, device=device)
+
+
+def get_tmp_tensor_for_size_no_kernels(size: int, device: torch.device) -> torch.Tensor:
+    return torch.empty((size,), dtype=torch.uint8, device=device)
+
+
+def get_tmp_expand_size(size: int) -> int:
+    return _kernels.sgmv_cutlass_tmp_size(size)
+
+
+def get_tmp_tensors(
+    nsegments: int, lora_rank: int, device: torch.device
+) -> Tuple[torch.Tensor, torch.Tensor]:
+    use_cutlass = use_cutlass_shrink(lora_rank) and has_sgmv()
+    has_sgmv_available = has_sgmv()
+
+    if use_cutlass:
+        tmp = get_tmp_tensor_for_size(nsegments, device)
+        return tmp, tmp
+    elif has_sgmv_available:
+        return get_tmp_tensor(device), get_tmp_tensor_for_size(nsegments, device)
+    else:
+        tmp = get_tmp_tensor_for_size(nsegments, device)
+        return tmp, tmp
+
+
+def lora_a_sgmv_cutlass(
+    x: torch.Tensor,
+    tmp: torch.Tensor,
+    wa_ptr: torch.Tensor,
+    s_start: torch.IntTensor,
+    s_end: torch.IntTensor,
+    layer_idx: int,
+    lora_rank: int,
+) -> torch.Tensor:
+    v = torch.zeros((x.size(0), lora_rank), dtype=x.dtype, device=x.device)
+    if MIN_RANK_CUSTOM <= lora_rank <= MAX_RANK_CUSTOM:
+        _kernels.sgmv_shrink(v, x, wa_ptr, s_start, s_end, tmp, layer_idx)
+    else:
+        _kernels.sgmv_cutlass(v, x, wa_ptr, s_start, s_end, tmp, layer_idx)
+    return v
+
+
+def lora_b_sgmv_cutlass(
+    y: torch.Tensor,
+    v: torch.Tensor,
+    tmp: torch.Tensor,
+    wb_ptr: torch.Tensor,
+    s_start: torch.IntTensor,
+    s_end: torch.IntTensor,
+    layer_idx: int,
+):
+    _kernels.sgmv_cutlass(y, v, wb_ptr, s_start, s_end, tmp, layer_idx)
+
+
+"""
+Semantics:
+    y[i] += (
+        x[i].unsqueeze(0)
+        @ wa_T_all[indices[i], layer_idx, :, :].transpose(-1, -2)
+        @ wb_T_all[indices[i], layer_idx, :, :].transpose(-1, -2)
+        * scale
+    ).squeeze(0)
+
+Args:
+    y: Shape: `[B, H2]`. Output vectors. Will be changed in-place.
+    v: Shape: `[B, R]`. Temporary vector.
+    x: Shape: `[B, H1]`. Input vectors.
+    wa_T_all: Shape: `[None, L, R, H1]`. All of the transposed LoRA A matrices.
+    wb_T_all: Shape: `[None, L, H2, R]`. All of the transposed LoRA B matrices.
+    indicies: Shape: `[B]`. Indices of the LoRA weights.
+    layer_idx: Layer index of LoRA weights.
+    scale: Scaling factor.
+"""
+
+
+def add_lora_a_bgmv(
+    v: torch.Tensor,
+    x: torch.Tensor,
+    wa_T_all: torch.Tensor,
+    indicies: torch.LongTensor,
+    layer_idx: int,
+):
+    _kernels.dispatch_bgmv(v, x, wa_T_all, indicies, layer_idx, 1.0)
+
+
+def add_lora_b_bgmv(
+    y: torch.Tensor,
+    v: torch.Tensor,
+    wb_T_all: torch.Tensor,
+    indicies: torch.LongTensor,
+    layer_idx: int,
+):
+    _kernels.dispatch_bgmv(y, v, wb_T_all, indicies, layer_idx, 1.0)
+
+
+def segmented_matmul(
+    y: torch.Tensor,
+    x: torch.Tensor,
+    w: List[torch.Tensor],
+    b: List[torch.Tensor],
+    s_start: torch.IntTensor,
+    s_end: torch.IntTensor,
+):
+    for i in range(len(w)):
+        if s_end[i] - s_start[i] <= 0:
+            continue
+
+        xi = x[s_start[i] : s_end[i]]
+        wi = w[i]
+        bi = b[i]
+        y[s_start[i] : s_end[i]] = F.linear(xi, wi, bi)
diff --git a/server/text_generation_server/utils/tokens.py b/server/text_generation_server/utils/tokens.py
index d2dca8b9..aa4d1fdb 100644
--- a/server/text_generation_server/utils/tokens.py
+++ b/server/text_generation_server/utils/tokens.py
@@ -4,7 +4,6 @@ import os
 import re
 from typing import List, Optional, Tuple, Set, Union
 
-import math
 import torch
 from text_generation_server.pb import generate_pb2
 from text_generation_server.pb.generate_pb2 import FinishReason, GrammarType
@@ -23,22 +22,21 @@ from text_generation_server.utils.logits_process import (
 )
 from text_generation_server.utils.watermark import WatermarkLogitsProcessor
 from transformers import PreTrainedTokenizerBase, RepetitionPenaltyLogitsProcessor
-from transformers.utils import to_py_obj
 
 
 class NextTokenChooser:
     def __init__(
         self,
-        watermark=False,
-        temperature=1.0,
-        repetition_penalty=1.0,
-        frequency_penalty=0.0,
-        top_k=None,
-        top_p=None,
-        typical_p=None,
-        do_sample=False,
-        seed=0,
-        device="cpu",
+        watermark: bool = False,
+        temperature: float = 1.0,
+        repetition_penalty: float = 1.0,
+        frequency_penalty: float = 0.0,
+        top_k: Optional[int] = None,
+        top_p: Optional[float] = None,
+        typical_p: Optional[float] = None,
+        do_sample: bool = False,
+        seed: int = 0,
+        device: str = "cpu",
         tokenizer: Optional[PreTrainedTokenizerBase] = None,
         grammar: str = "",
         grammar_type: GrammarType = GrammarType.GRAMMAR_TYPE_NONE,
@@ -71,7 +69,9 @@ class NextTokenChooser:
             or (typical_p is not None and typical_p < 1.0)
         )
         if has_warpers:
-            self.static_warper = static_warper(temperature=temperature, top_k=top_k, top_p=top_p, typical_p=typical_p)
+            self.static_warper = static_warper(
+                temperature=temperature, top_k=top_k, top_p=top_p, typical_p=typical_p
+            )
         else:
             self.static_warper = None
 
@@ -254,7 +254,7 @@ class HeterogeneousNextTokenChooser:
         tokenizer: PreTrainedTokenizerBase,
         grammars: List[str],
         grammar_types: List[int],
-        fsm_grammar_states: List[int],
+        fsm_grammar_states:List[int],
         quantization_enabled: bool,
     ):
         warpers = []
@@ -650,8 +650,9 @@ def batch_top_tokens(
     top_n_indices = (logprobs >= nth_highest).nonzero()
     _, top_n_ishes = torch.unique_consecutive(top_n_indices[:, 0], return_counts=True)
 
+    k = 1 if top_n_ishes.numel() == 0 else top_n_ishes.max()
     # Take a new topk for these new max n values
-    top_k = torch.topk(logprobs, k=top_n_ishes.max(), dim=1, sorted=True)
+    top_k = torch.topk(logprobs, k=k, dim=1, sorted=True)
 
     top_n_ishes = top_n_ishes.tolist()
     top_indices = top_k.indices.tolist()
diff --git a/server/text_generation_server/utils/watermark.py b/server/text_generation_server/utils/watermark.py
index 7f4bf367..5092b076 100644
--- a/server/text_generation_server/utils/watermark.py
+++ b/server/text_generation_server/utils/watermark.py
@@ -39,12 +39,16 @@ class WatermarkLogitsProcessor(LogitsProcessor):
 
     def _seed_rng(self, input_ids: Union[List[int], torch.LongTensor]):
         if isinstance(input_ids, list):
-            assert len(input_ids) >= 1, "requires at least a 1 token prefix sequence to seed rng"
+            assert (
+                len(input_ids) >= 1
+            ), "requires at least a 1 token prefix sequence to seed rng"
             prev_token = input_ids[-1]
         else:
             assert len(input_ids) == 1
             input_ids = input_ids[0]
-            assert input_ids.shape[-1] >= 1, "requires at least a 1 token prefix sequence to seed rng"
+            assert (
+                input_ids.shape[-1] >= 1
+            ), "requires at least a 1 token prefix sequence to seed rng"
             prev_token = input_ids[-1].item()
         self.rng.manual_seed(self.hash_key * prev_token)
 
@@ -63,7 +67,9 @@ class WatermarkLogitsProcessor(LogitsProcessor):
         return greenlist_ids
 
     @staticmethod
-    def _calc_greenlist_mask(scores: torch.FloatTensor, greenlist_token_ids) -> torch.BoolTensor:
+    def _calc_greenlist_mask(
+        scores: torch.FloatTensor, greenlist_token_ids
+    ) -> torch.BoolTensor:
         green_tokens_mask = torch.zeros_like(scores)
         green_tokens_mask[-1, greenlist_token_ids] = 1
         final_mask = green_tokens_mask.bool()
@@ -76,9 +82,15 @@ class WatermarkLogitsProcessor(LogitsProcessor):
         scores[greenlist_mask] = scores[greenlist_mask] + greenlist_bias
         return scores
 
-    def __call__(self, input_ids: Union[List[int], torch.LongTensor], scores: torch.FloatTensor) -> torch.FloatTensor:
-        greenlist_ids = self._get_greenlist_ids(input_ids, scores.shape[-1], scores.device)
-        green_tokens_mask = self._calc_greenlist_mask(scores=scores, greenlist_token_ids=greenlist_ids)
+    def __call__(
+        self, input_ids: Union[List[int], torch.LongTensor], scores: torch.FloatTensor
+    ) -> torch.FloatTensor:
+        greenlist_ids = self._get_greenlist_ids(
+            input_ids, scores.shape[-1], scores.device
+        )
+        green_tokens_mask = self._calc_greenlist_mask(
+            scores=scores, greenlist_token_ids=greenlist_ids
+        )
 
         scores = self._bias_greenlist_logits(
             scores=scores, greenlist_mask=green_tokens_mask, greenlist_bias=self.delta
diff --git a/server/text_generation_server/utils/weights.py b/server/text_generation_server/utils/weights.py
index 6af7d3fb..75e01f7c 100644
--- a/server/text_generation_server/utils/weights.py
+++ b/server/text_generation_server/utils/weights.py
@@ -1,12 +1,139 @@
-import os
-from pathlib import Path
-from typing import List, Dict, Optional, Tuple
-from safetensors import safe_open, SafetensorError
 import torch
-from loguru import logger
-from huggingface_hub import hf_hub_download
-import json
-from text_generation_server.utils.log import log_once
+
+from abc import ABC, abstractmethod
+from contextlib import contextmanager
+from pathlib import Path
+from typing import Dict, List, Optional, Union, Type
+from safetensors import safe_open
+from dataclasses import dataclass
+
+from text_generation_server.utils.import_utils import SYSTEM
+
+
+class WeightsLoader(ABC):
+    """
+    Instances of this type implement higher-level weight loading.
+
+    At a low-level, every weight is stored in the Safetensors format.
+    The interpretation of weights may be different however, for instance
+    could be packed, quantized weights. Loaders are responsible for
+    interpreting the raw tensors, sharding tensors in a manner compatible
+    with the format, etc.
+    """
+
+    @abstractmethod
+    def get_weights(self, weights: "Weights", prefix: str):
+        """
+        Get weights at the given prefix and apply without tensor paralllism.
+        """
+        ...
+
+    @abstractmethod
+    def get_weights_col_packed(
+        self,
+        weights: "Weights",
+        prefix: str,
+        block_sizes: Union[int, List[int]],
+    ):
+        """
+        Get the packed weights at the given prefix with column-splitting for
+        tensor parallelism. This method should be used when multiple different
+        weights are packed into a tensor, for instance, query/key/value
+        weights or a gate/up projection.
+
+        The `block_sizes` determines the proportions of the packed tensors.
+        The columns are split in equally sized blocks when `block_sizes` is an
+        `int`, or in blocks proportional given to the sizes. For instance
+        `[2, 1, 1]` will divide an input with dimensionality `1024` in
+        `[512, 256, 256]`.
+        """
+        ...
+
+    def get_weights_col(self, weights: "Weights", prefix: str):
+        """
+        Get weights at the given prefix and apply column-splitting for tensor
+        paralllism.
+        """
+        return weights.get_multi_weights_col([prefix], 0)
+
+    @abstractmethod
+    def get_multi_weights_col(self, weights: "Weights", prefixes: List[str], dim: int):
+        """
+        Get the weights at the given prefixes, column-split them for tensor
+        parallelim, and then concatenate the weights along the given dimension.
+        """
+        ...
+
+    @abstractmethod
+    def get_weights_row(self, weights: "Weights", prefix: str):
+        """
+        Get the weights at the given prefix and apply row-splitting for tensor
+        parallism.
+        """
+        ...
+
+
+class Weight(ABC):
+    """Instances of this type implement unquantized/quantized/to-be
+    quantized weights."""
+
+    @abstractmethod
+    def get_linear(self, bias: torch.Tensor):
+        """Create a linear layer from this weight."""
+        ...
+
+
+@dataclass
+class UnquantizedWeight(Weight):
+    weight: torch.Tensor
+
+    def get_linear(self, bias: torch.Tensor):
+        from text_generation_server.layers.linear import FastLinear, FastLinearROCm
+
+        if SYSTEM == "rocm":
+            return FastLinearROCm(self.weight, bias)
+        else:
+            return FastLinear(self.weight, bias)
+
+
+class DefaultWeightsLoader(WeightsLoader):
+    """Weight loader that loads (unquantized) Torch tensors."""
+
+    def __init__(self, weight_class: Type[UnquantizedWeight]):
+        """Create a loader. Weights will be wrapped using the given `weights_class`,
+        normally this will be `UnquantizedWeight`, but a quantizer-specific class
+        such as `Fp8Weight` can be used to quantize the weights during loading.
+        """
+        self.weight_class = weight_class
+
+    """
+    Loader that uses tensors as-is with the exception of applying sharding
+    and/or concatenation.
+    """
+
+    def get_weights(self, weights: "Weights", prefix: str):
+        return weights.get_tensor(f"{prefix}.weight")
+
+    def get_weights_col_packed(
+        self,
+        weights: "Weights",
+        prefix: str,
+        block_sizes: Union[int, List[int]],
+    ):
+        return self.weight_class(
+            weights.get_packed_sharded(
+                f"{prefix}.weight", dim=0, block_sizes=block_sizes
+            ),
+        )
+
+    def get_multi_weights_col(self, weights: "Weights", prefixes: List[str], dim: int):
+        w = [weights.get_sharded(f"{p}.weight", dim=0) for p in prefixes]
+        return self.weight_class(torch.cat(w, dim=dim))
+
+    def get_weights_row(self, weights: "Weights", prefix: str):
+        return self.weight_class(
+            weights.get_sharded(f"{prefix}.weight", dim=1),
+        )
 
 
 class Weights:
@@ -16,6 +143,7 @@ class Weights:
         device,
         dtype,
         process_group,
+        weights_loader: WeightsLoader,
         aliases: Optional[Dict[str, List[str]]] = None,
         prefix: Optional[str] = None,
     ):
@@ -36,6 +164,7 @@ class Weights:
         self.dtype = dtype
         self.process_group = process_group
         self.prefix = prefix
+        self.weights_loader = weights_loader
         self._handles = {}
 
     def _get_handle(self, filename):
@@ -68,22 +197,41 @@ class Weights:
         slice_ = f.get_slice(tensor_name)
         return slice_
 
+    def _has_tensor(self, tensor_name: str):
+        try:
+            self.get_filename(tensor_name)
+        except Exception:
+            return False
+        return True
+
     def get_shape(self, tensor_name: str):
         return self._get_slice(tensor_name).get_shape()
 
-    def get_tensor(self, tensor_name: str, to_device=True):
+    def get_tensor(self, tensor_name: str, to_device=True, to_dtype=True):
         filename, tensor_name = self.get_filename(tensor_name)
         f = self._get_handle(filename)
         tensor = f.get_tensor(tensor_name)
         # Special case for gptq which shouldn't convert
-        # u4 which are disguised as int32
-        if tensor.dtype not in [torch.int32, torch.int64]:
+        # u4 which are disguised as int32. Exl2 uses int16
+        # as well. FP8 uses torch.float8_e4m3fn
+        if (
+            tensor.dtype
+            not in [
+                torch.float8_e4m3fn,
+                torch.int16,
+                torch.int32,
+                torch.int64,
+            ]
+            and to_dtype
+        ):
             tensor = tensor.to(dtype=self.dtype)
         if to_device:
             tensor = tensor.to(device=self.device)
         return tensor
 
-    def get_partial_sharded(self, tensor_name: str, dim: int):
+    def get_partial_sharded(
+        self, tensor_name: str, dim: int, to_device=True, to_dtype=True
+    ):
         filename, tensor_name = self.get_filename(tensor_name)
         f = self._get_handle(filename)
         slice_ = f.get_slice(tensor_name)
@@ -102,13 +250,18 @@ class Weights:
         else:
             raise NotImplementedError("Let's make that generic when needed")
         # Special case for gptq which shouldn't convert
-        # u4 which are disguised as int32
-        if tensor.dtype != torch.int32:
+        # u4 which are disguised as int32. exl2 uses int16.
+        # FP8 uses torch.float8_e4m3fn.
+        if (
+            tensor.dtype not in (torch.float8_e4m3fn, torch.int16, torch.int32)
+            and to_dtype
+        ):
             tensor = tensor.to(dtype=self.dtype)
-        tensor = tensor.to(device=self.device)
+        if to_device:
+            tensor = tensor.to(device=self.device)
         return tensor
 
-    def get_sharded(self, tensor_name: str, dim: int):
+    def get_sharded(self, tensor_name: str, dim: int, to_device=True, to_dtype=True):
         filename, tensor_name = self.get_filename(tensor_name)
         f = self._get_handle(filename)
         slice_ = f.get_slice(tensor_name)
@@ -117,153 +270,106 @@ class Weights:
         assert (
             size % world_size == 0
         ), f"The choosen size {size} is not compatible with sharding on {world_size} shards"
-        return self.get_partial_sharded(tensor_name, dim)
+        return self.get_partial_sharded(
+            tensor_name, dim, to_device=to_device, to_dtype=to_dtype
+        )
+
+    def get_packed_sharded(
+        self,
+        tensor_name: str,
+        dim: int,
+        block_sizes: Union[int, List[int]],
+        to_dtype=True,
+    ) -> torch.Tensor:
+        """
+        Get a shard from a tensor that packs multiple tensors.
+
+        When a tensor packs multiple tensors (such as QKV or an up
+        projection + gate projection), sharding with `get_sharded` is not
+        safe since it would not split the packed tensors across shards.
+
+        This method shards a tensor, such that the packed tensors are
+        split across shards.
+
+        The columns are split in equally sized blocks when blocks is an `int`, or
+        in blocks proportional given to the sizes. For instance `[2, 1, 1]` will
+        divide an input with dimensionality `1024` in `[512, 256, 256]`. This is
+        convenient for e.g. splitting QKV without knowing the storage details of
+        quantized weights.
+        """
+        slice_ = self._get_slice(tensor_name)
+        total_size = slice_.get_shape()[dim]
+        block_sizes = _blocks_to_block_sizes(total_size=total_size, blocks=block_sizes)
 
-    def _get_qweight(self, name: str):
-        slice_ = self._get_slice(name)
-        total_size = slice_.get_shape()[1]
-        assert total_size % 3 == 0, "Prepacked quantized qkv is not divisible by 3"
-        single_size = total_size // 3
         world_size = self.process_group.size()
         rank = self.process_group.rank()
 
-        assert (
-            single_size % world_size == 0
-        ), f"Prepacked quantized qkv cannot be sharded across {world_size} shards"
-        block_size = single_size // world_size
-        start = rank * block_size
-        stop = (rank + 1) * block_size
-        q = slice_[:, start:stop]
-        k = slice_[:, start + single_size : stop + single_size]
-        v = slice_[:, start + 2 * single_size : stop + 2 * single_size]
-        weight = torch.cat([q, k, v], dim=1)
-        weight = weight.to(device=self.device)
-        return weight
-
-    def get_weights_col_packed_qkv(self, prefix: str, quantize: str):
-        return self.get_weights_col_packed(prefix, quantize, 3)
-
-    def get_weights_col_packed_gate_up(self, prefix: str, quantize: str):
-        return self.get_weights_col_packed(prefix, quantize, 2)
-
-    def get_weights_col_packed(self, prefix: str, quantize: str, blocks: int):
-        """
-        Highly specific when the underlying tensor is a simple cat of Q,K,V instead of being
-        already alternating Q,K,V within the main tensor
-        """
-        if quantize in ["gptq", "awq"]:
-            try:
-                qweight = self._get_qweight(f"{prefix}.qweight")
-            except RuntimeError:
-                raise RuntimeError(
-                    f"Cannot load `{quantize}` weight, make sure the model is already quantized."
-                )
-
-            bits, groupsize, _, quant_method = self._get_gptq_params()
-
-            qzeros = self._get_qweight(f"{prefix}.qzeros")
-            scales = self._get_qweight(f"{prefix}.scales")
-            scales = scales.to(dtype=self.dtype)
-
-            if quantize == "gptq" and quant_method == "gptq":
-                g_idx = self.get_tensor(f"{prefix}.g_idx")
-            elif quantize == "gptq" and quant_method == "awq":
-                log_once(
-                    logger.info, "Converting AWQ model to Exllama/GPTQ packing format."
-                )
-                from text_generation_server.layers.awq.conversion_utils import (
-                    fast_awq_to_gptq,
-                )
-
-                qweight, qzeros = fast_awq_to_gptq(qweight, qzeros)
-                g_idx = (
-                    torch.arange(qweight.shape[0] * (32 // bits), device=qweight.device)
-                    // groupsize
-                ).to(dtype=torch.int32)
-            else:
-                g_idx = None
-
-            weight = (qweight, qzeros, scales, g_idx, bits, groupsize, False)
-        else:
-            slice_ = self._get_slice(f"{prefix}.weight")
-            total_size = slice_.get_shape()[0]
-            assert total_size % blocks == 0, f"Prepacked is not divisible by {blocks}"
-            single_size = total_size // blocks
-            world_size = self.process_group.size()
-            rank = self.process_group.rank()
-
+        tensors = []
+        block_offset = 0
+        for block_size in block_sizes:
             assert (
-                single_size % world_size == 0
-            ), f"Prepacked qkv cannot be sharded across {world_size} shards"
-            block_size = single_size // world_size
-            start = rank * block_size
-            stop = (rank + 1) * block_size
-            tensors = []
-            for i in range(blocks):
-                tensor = slice_[start + i * single_size : stop + i * single_size]
-                tensors.append(tensor)
-            weight = torch.cat(tensors, dim=0)
-            weight = weight.to(device=self.device)
-            weight = weight.to(dtype=self.dtype)
-        return weight
-
-    def get_multi_weights_col(self, prefixes: List[str], quantize: str, dim: int):
-        if quantize in ["gptq", "awq"]:
-            try:
-                qweight = torch.cat(
-                    [self.get_sharded(f"{p}.qweight", dim=1) for p in prefixes], dim=1
-                )
-            except RuntimeError:
-                raise RuntimeError(
-                    f"Cannot load `{quantize}` weight, make sure the model is already quantized"
-                )
-
-            qzeros = torch.cat(
-                [self.get_sharded(f"{p}.qzeros", dim=1) for p in prefixes], dim=1
-            )
-            scales = torch.cat(
-                [self.get_sharded(f"{p}.scales", dim=1) for p in prefixes], dim=1
-            )
-
-            bits, groupsize, desc_act, quant_method = self._get_gptq_params()
-
-            from text_generation_server.layers.gptq import HAS_EXLLAMA
-
-            use_exllama = (
-                bits == 4 and HAS_EXLLAMA and quantize == "gptq" and not desc_act
-            )
-
-            if quantize == "gptq" and quant_method == "gptq":
-                w = [self.get_tensor(f"{p}.g_idx") for p in prefixes]
-                for w2 in w[1:]:
-                    torch.testing.assert_close(w2, w[0])
-                g_idx = w[0]
-            elif quantize == "gptq" and quant_method == "awq":
-                log_once(
-                    logger.info, "Converting AWQ model to Exllama/GPTQ packing format."
-                )
-                from text_generation_server.layers.awq.conversion_utils import (
-                    fast_awq_to_gptq,
-                )
-
-                qweight, qzeros = fast_awq_to_gptq(qweight, qzeros)
-                if use_exllama:
-                    g_idx = None
-                else:
-                    g_idx = (
-                        torch.arange(
-                            qweight.shape[0] * (32 // bits), device=qweight.device
-                        )
-                        // groupsize
-                    ).to(dtype=torch.int32)
+                block_size % world_size == 0
+            ), f"Prepacked tensor cannot be sharded across {world_size} shards"
+            shard_block_size = block_size // world_size
+            start = rank * shard_block_size
+            stop = (rank + 1) * shard_block_size
+            if dim == 0:
+                tensor = slice_[block_offset + start : block_offset + stop]
+            elif dim == 1:
+                tensor = slice_[:, block_offset + start : block_offset + stop]
             else:
-                g_idx = None
+                raise NotImplementedError("Currently only dim=0 or dim=1 is supported")
+            tensors.append(tensor)
+            block_offset += block_size
+        tensor = torch.cat(tensors, dim=dim)
+        tensor = tensor.to(device=self.device)
 
-            weight = (qweight, qzeros, scales, g_idx, bits, groupsize, use_exllama)
-        else:
-            w = [self.get_sharded(f"{p}.weight", dim=0) for p in prefixes]
-            weight = torch.cat(w, dim=dim)
-        return weight
+        # Avoid casting quantizer dtypes.
+        if (
+            tensor.dtype
+            not in [
+                torch.float8_e4m3fn,
+                torch.int16,
+                torch.int32,
+                torch.int64,
+            ]
+            and to_dtype
+        ):
+            tensor = tensor.to(dtype=self.dtype)
+
+        return tensor
+
+    def get_weights(self, prefix: str):
+        return self.weights_loader.get_weights(self, prefix)
+
+    def get_weights_col_packed_qkv(
+        self,
+        prefix: str,
+        num_heads: int,
+        num_key_value_heads: int,
+    ):
+        return self.get_weights_col_packed(
+            prefix, [num_heads, num_key_value_heads, num_key_value_heads]
+        )
+
+    def get_weights_col_packed_gate_up(self, prefix: str):
+        return self.get_weights_col_packed(prefix, 2)
+
+    def get_weights_col_packed(self, prefix: str, block_sizes: Union[int, List[int]]):
+        """
+        The columns are split in equally sized blocks when blocks is an `int`, or
+        in blocks proportional given to the sizes. For instance `[2, 1, 1]` will
+        divide an input with dimensionality `1024` in `[512, 256, 256]`. This is
+        convenient for e.g. splitting QKV without knowing the storage details of
+        quantized weights.
+        """
+        return self.weights_loader.get_weights_col_packed(self, prefix, block_sizes)
+
+    def get_weights_col(self, prefix: str):
+        return self.weights_loader.get_weights_col(self, prefix)
+
+    def get_multi_weights_col(self, prefixes: List[str], dim: int):
+        return self.weights_loader.get_multi_weights_col(self, prefixes, dim)
 
     def get_tensor_shard(self, var, dim):
         world_size = self.process_group.size()
@@ -281,173 +387,51 @@ class Weights:
         tensor = tensor.to(device=self.device)
         return tensor
 
-    def get_multi_weights_row(self, prefix: str, quantize: str):
-        if quantize == "gptq":
-            use_exllama = True
-            bits, groupsize, desc_act, quant_method = self._get_gptq_params()
+    def get_weights_row(self, prefix: str):
+        return self.weights_loader.get_weights_row(self, prefix)
 
-            if bits != 4:
-                use_exllama = False
+    @contextmanager
+    def use_loader(self, weights_loader: WeightsLoader):
+        """
+        This method is a context manager that can be used to use `Weights` with
+        a different loader for the duration of the context.
+        """
 
-            if desc_act:
-                log_once(logger.warning, "Disabling exllama because desc_act=True")
-                use_exllama = False
-
-            try:
-                qweight = self.get_sharded(f"{prefix}.qweight", dim=0)
-            except RuntimeError:
-                raise RuntimeError(
-                    "Cannot load `gptq` weight, make sure the model is already quantized, or quantize it with `text-generation-server quantize ORIGINAL_MODEL_ID NEW_MODEL_ID`"
-                )
-
-            if quant_method == "gptq":
-                g_idx = self.get_sharded(f"{prefix}.g_idx", dim=0)
-            elif quant_method == "awq":
-                g_idx = None
-
-            if self.process_group.size() > 1:
-                if g_idx is not None:
-                    if (
-                        not torch.equal(
-                            g_idx.cpu(),
-                            torch.tensor(
-                                [i // groupsize for i in range(g_idx.shape[0])],
-                                dtype=torch.int32,
-                            ),
-                        )
-                        and not (g_idx == 0).all()
-                    ):
-                        # Exllama implementation does not support row tensor parallelism with act-order, as
-                        # it would require to reorder input activations that are split unto several GPUs
-                        use_exllama = False
-
-            from text_generation_server.layers.gptq import HAS_EXLLAMA, CAN_EXLLAMA
-
-            if use_exllama:
-                if not HAS_EXLLAMA:
-                    if CAN_EXLLAMA:
-                        log_once(
-                            logger.warning,
-                            "Exllama GPTQ cuda kernels (which are faster) could have been used, but are not currently installed, try using BUILD_EXTENSIONS=True",
-                        )
-                    use_exllama = False
-                else:
-                    log_once(logger.info, f"Using exllama kernels v{HAS_EXLLAMA}")
-
-            if use_exllama and groupsize != -1:
-                qzeros = self.get_sharded(f"{prefix}.qzeros", dim=0)
-                scales = self.get_sharded(f"{prefix}.scales", dim=0)
-            else:
-                qzeros = self.get_tensor(f"{prefix}.qzeros")
-                scales = self.get_tensor(f"{prefix}.scales")
-
-            if use_exllama and g_idx is not None:
-                g_idx = g_idx - g_idx[0]
-
-            if quant_method == "awq":
-                log_once(
-                    logger.info, "Converting AWQ model to Exllama/GPTQ packing format."
-                )
-                from text_generation_server.layers.awq.conversion_utils import (
-                    fast_awq_to_gptq,
-                )
-
-                qweight, qzeros = fast_awq_to_gptq(qweight, qzeros)
-                if use_exllama:
-                    g_idx = None
-                else:
-                    g_idx = (
-                        torch.arange(
-                            qweight.shape[0] * (32 // bits), device=qweight.device
-                        )
-                        // groupsize
-                    ).to(dtype=torch.int32)
-
-            weight = (qweight, qzeros, scales, g_idx, bits, groupsize, use_exllama)
-        elif quantize == "awq":
-            bits, groupsize, _, _ = self._get_gptq_params()
-
-            try:
-                qweight = self.get_sharded(f"{prefix}.qweight", dim=0)
-            except RuntimeError:
-                raise RuntimeError(
-                    "Cannot load `awq` weight, make sure the model is already quantized"
-                )
-
-            qzeros = self.get_sharded(f"{prefix}.qzeros", dim=0)
-            scales = self.get_sharded(f"{prefix}.scales", dim=0)
-            g_idx = None
-            use_exllama = False
-
-            weight = (qweight, qzeros, scales, g_idx, bits, groupsize, use_exllama)
-        else:
-            weight = self.get_sharded(f"{prefix}.weight", dim=1)
-        return weight
-
-    def _get_gptq_params(self) -> Tuple[int, int, int, str]:
+        old_loader = self.weights_loader
+        self.weights_loader = weights_loader
         try:
-            bits = self.get_tensor("gptq_bits").item()
-            groupsize = self.get_tensor("gptq_groupsize").item()
-            desc_act = False
-            quant_method = "gptq"
-        except (SafetensorError, RuntimeError) as e:
-            try:
-                bits = self.gptq_bits
-                groupsize = self.gptq_groupsize
-                desc_act = getattr(self, "gptq_desc_act", False)
-                quant_method = getattr(self, "quant_method", "gptq")
-            except Exception:
-                raise e
+            yield
+        finally:
+            self.weights_loader = old_loader
 
-        return bits, groupsize, desc_act, quant_method
+    @property
+    def loader(self):
+        return self.weights_loader
 
-    def _set_gptq_params(self, model_id, revision):
-        filename = "config.json"
-        try:
-            if os.path.exists(os.path.join(model_id, filename)):
-                filename = os.path.join(model_id, filename)
-            else:
-                filename = hf_hub_download(
-                    model_id, filename=filename, revision=revision
-                )
-            with open(filename, "r") as f:
-                data = json.load(f)
-            self.gptq_bits = data["quantization_config"]["bits"]
-            self.gptq_groupsize = data["quantization_config"]["group_size"]
-            # Order is important here, desc_act is missing on some real models
-            self.quant_method = data["quantization_config"]["quant_method"]
-            self.gptq_desc_act = data["quantization_config"]["desc_act"]
-        except Exception:
-            filename = "quantize_config.json"
-            try:
-                if os.path.exists(os.path.join(model_id, filename)):
-                    filename = os.path.join(model_id, filename)
-                else:
-                    filename = hf_hub_download(
-                        model_id, filename=filename, revision=revision
-                    )
-                with open(filename, "r") as f:
-                    data = json.load(f)
-                self.gptq_bits = data["bits"]
-                self.gptq_groupsize = data["group_size"]
-                self.gptq_desc_act = data["desc_act"]
-                if "version" in data and data["version"] == "GEMM":
-                    self.quant_method = "awq"
-            except Exception:
-                filename = "quant_config.json"
-                try:
-                    if os.path.exists(os.path.join(model_id, filename)):
-                        filename = os.path.join(model_id, filename)
-                    else:
-                        filename = hf_hub_download(
-                            model_id, filename=filename, revision=revision
-                        )
-                    with open(filename, "r") as f:
-                        data = json.load(f)
-                    self.gptq_bits = data["w_bit"]
-                    self.gptq_groupsize = data["q_group_size"]
-                    self.gptq_desc_act = data["desc_act"]
-                    if "version" in data and data["version"] == "GEMM":
-                        self.quant_method = "awq"
-                except Exception:
-                    pass
+
+def _blocks_to_block_sizes(total_size: int, blocks: Union[int, List[int]]) -> List[int]:
+    """
+    Convert block count or proportions to block sizes.
+
+    This function accepts
+
+    - The number of blocks (int), in which case the block size is
+      total_size//blocks; or
+    - A list of block sizes (List[int]).
+
+    In the latter case, if sum(blocks) < total_size, the ratios between
+    the block sizes will be preserved. For instance, if blocks is
+    [2, 1, 1] and total_size is 1024, the returned block sizes are
+    [512, 256, 256].
+    """
+    if isinstance(blocks, list):
+        total_blocks = sum(blocks)
+        assert (
+            total_size % total_blocks == 0
+        ), f"Cannot split {total_size} in proportional blocks: {blocks}"
+        part_size = total_size // total_blocks
+        return [part_size * block for block in blocks]
+    else:
+        assert total_size % blocks == 0, f"Prepacked is not divisible by {blocks}"
+        single_size = total_size // blocks
+        return [single_size] * blocks
diff --git a/tgi-entrypoint.sh b/tgi-entrypoint.sh
new file mode 100755
index 00000000..ea94dcd9
--- /dev/null
+++ b/tgi-entrypoint.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+ldconfig 2>/dev/null || echo 'unable to refresh ld cache, not a big deal in most cases'
+
+text-generation-launcher $@
diff --git a/update_doc.py b/update_doc.py
index 5da81c72..3fb0d314 100644
--- a/update_doc.py
+++ b/update_doc.py
@@ -1,11 +1,13 @@
 import subprocess
 import argparse
 import ast
+import json
+import os
 
 TEMPLATE = """
 # Supported Models and Hardware
 
-Text Generation Inference enables serving optimized models on specific hardware for the highest performance. The following sections list which models are hardware are supported.
+Text Generation Inference enables serving optimized models on specific hardware for the highest performance. The following sections list which models (VLMs & LLMs) are supported.
 
 ## Supported Models
 
@@ -61,7 +63,7 @@ def check_cli(check: bool):
     final_doc += f"## {header}\n```shell\n{rendered_block}\n```\n"
     block = []
 
-    filename = "docs/source/basic_tutorials/launcher.md"
+    filename = "docs/source/reference/launcher.md"
     if check:
         with open(filename, "r") as f:
             doc = f.read()
@@ -122,6 +124,71 @@ def check_supported_models(check: bool):
             f.write(final_doc)
 
 
+def get_openapi_schema():
+    try:
+        output = subprocess.check_output(["text-generation-router", "print-schema"])
+        return json.loads(output)
+    except subprocess.CalledProcessError as e:
+        print(f"Error running text-generation-router print-schema: {e}")
+        raise SystemExit(1)
+    except json.JSONDecodeError:
+        print("Error: Invalid JSON received from text-generation-router print-schema")
+        raise SystemExit(1)
+
+
+def check_openapi(check: bool):
+    new_openapi_data = get_openapi_schema()
+    filename = "docs/openapi.json"
+    tmp_filename = "openapi_tmp.json"
+
+    with open(tmp_filename, "w") as f:
+        json.dump(new_openapi_data, f, indent=2)
+
+    if check:
+        diff = subprocess.run(
+            [
+                "diff",
+                # allow for trailing whitespace since it's not significant
+                # and the precommit hook will remove it
+                "--ignore-trailing-space",
+                tmp_filename,
+                filename,
+            ],
+            capture_output=True,
+        ).stdout.decode("utf-8")
+        os.remove(tmp_filename)
+
+        if diff:
+            print(diff)
+            raise Exception(
+                "OpenAPI documentation is not up-to-date, run `python update_doc.py` in order to update it"
+            )
+
+    else:
+        os.rename(tmp_filename, filename)
+        print("OpenAPI documentation updated.")
+    p = subprocess.run(
+        [
+            "redocly",
+            # allow for trailing whitespace since it's not significant
+            # and the precommit hook will remove it
+            "lint",
+            filename,
+        ],
+        capture_output=True,
+    )
+    errors = p.stderr.decode("utf-8")
+    # The openapi specs fails on `exclusive_minimum` which is expected to be a boolean where
+    # utoipa outputs a value instead: https://github.com/juhaku/utoipa/issues/969
+    print(errors)
+    if p.returncode != 0:
+        print(errors)
+        raise Exception(
+            f"OpenAPI documentation is invalid, `redocly lint {filename}` showed some error:\n {errors}"
+        )
+    return True
+
+
 def main():
     parser = argparse.ArgumentParser()
     parser.add_argument("--check", action="store_true")
@@ -130,6 +197,7 @@ def main():
 
     check_cli(args.check)
     check_supported_models(args.check)
+    check_openapi(args.check)
 
 
 if __name__ == "__main__":