From 215ed3ad52651f76ca4326713ba9e4e5107323e5 Mon Sep 17 00:00:00 2001
From: drbh <david.richard.holtz@gmail.com>
Date: Mon, 5 Aug 2024 09:11:40 -0400
Subject: [PATCH 01/72] fix: attempt forward on flash attn2 to check hardware
 support (#2335)

* fix: attempt forward on flash attn2 to check hardware support

* fix: warn window_size_left when using flash attn 1

* fix: prefer version check over test op and avoid window_size_left if not flash attn2

* fix: improve condtional and error message

* fix: update sliding window conditional

* fix: simplify changes and revert model changes

* fix: avoid changing conditional

* fix: typo tweak
---
 server/text_generation_server/layers/attention/cuda.py | 4 ++++
 server/text_generation_server/models/__init__.py       | 3 +++
 2 files changed, 7 insertions(+)

diff --git a/server/text_generation_server/layers/attention/cuda.py b/server/text_generation_server/layers/attention/cuda.py
index dff742dc..2b898831 100644
--- a/server/text_generation_server/layers/attention/cuda.py
+++ b/server/text_generation_server/layers/attention/cuda.py
@@ -172,6 +172,10 @@ def paged_attention(
 
 
 try:
+    is_ampere_or_newer = major >= 8 and minor >= 0
+    if not is_ampere_or_newer:
+        raise ImportError("FlashAttention only supports Ampere GPUs or newer.")
+
     import flash_attn_2_cuda
 
     V2 = True
diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py
index 3dc24159..ae791ef8 100644
--- a/server/text_generation_server/models/__init__.py
+++ b/server/text_generation_server/models/__init__.py
@@ -484,6 +484,9 @@ def get_model(
         )
     sliding_window = config_dict.get("sliding_window", -1)
 
+    if max_input_tokens is not None and max_input_tokens <= sliding_window:
+        sliding_window = -1
+
     if (
         (sliding_window is not None and sliding_window != -1)
         and not SUPPORTS_WINDOWING

From dd47a3dac411a3b9896bada2983f5ec7014c1922 Mon Sep 17 00:00:00 2001
From: drbh <david.richard.holtz@gmail.com>
Date: Mon, 5 Aug 2024 12:36:44 -0400
Subject: [PATCH 02/72] feat: include local lora adapter loading docs (#2359)

---
 docs/source/conceptual/lora.md | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/docs/source/conceptual/lora.md b/docs/source/conceptual/lora.md
index 08df767c..cfc2109b 100644
--- a/docs/source/conceptual/lora.md
+++ b/docs/source/conceptual/lora.md
@@ -36,6 +36,18 @@ To use LoRA in TGI, when starting the server, you can specify the list of LoRA m
 LORA_ADAPTERS=predibase/customer_support,predibase/dbpedia
 ```
 
+additionally, you can specify the path to the LoRA models using the `LORA_ADAPTERS_PATH` environment variable. For example:
+
+```bash
+LORA_ADAPTERS=myadapter=/some/path/to/adapter,myadapter2=/another/path/to/adapter
+```
+
+note it's possible to mix adapter_ids with adapter_id=adapter_path e.g.
+
+```bash
+LORA_ADAPTERS=predibase/dbpedia,myadapter=/path/to/dir/
+```
+
 In the server logs, you will see the following message:
 
 ```txt

From 29b8d19cdf83d7fd26a4f17015228363a7163522 Mon Sep 17 00:00:00 2001
From: drbh <david.richard.holtz@gmail.com>
Date: Tue, 6 Aug 2024 07:49:53 -0400
Subject: [PATCH 03/72] fix: return the out tensor rather then the functions
 return value (#2361)

---
 server/text_generation_server/layers/attention/cuda.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/server/text_generation_server/layers/attention/cuda.py b/server/text_generation_server/layers/attention/cuda.py
index 2b898831..96b654d0 100644
--- a/server/text_generation_server/layers/attention/cuda.py
+++ b/server/text_generation_server/layers/attention/cuda.py
@@ -292,8 +292,7 @@ else:
                 )
 
         out = torch.empty_like(q)
-
-        return flash_attn_cuda.fwd(
+        flash_attn_cuda.fwd(
             q,
             k,
             v,
@@ -309,4 +308,5 @@ else:
             False,
             0,
             None,
-        )[0]
+        )
+        return out

From e11f5f1c383831a226e26bf3560d6cdade0ee914 Mon Sep 17 00:00:00 2001
From: drbh <david.richard.holtz@gmail.com>
Date: Tue, 6 Aug 2024 07:51:32 -0400
Subject: [PATCH 04/72] feat: implement a templated endpoint for visibility
 into chat requests (#2333)

* feat: implement a templated endpoint for visibility into chat requests

* feat: improve to tokenize too

* fix: adjust return type

* feat: simplify prepare_chat_input logic and adjust start stop chars
---
 router/src/lib.rs    |   6 ++
 router/src/server.rs | 210 ++++++++++++++++++++++++++++++-------------
 2 files changed, 156 insertions(+), 60 deletions(-)

diff --git a/router/src/lib.rs b/router/src/lib.rs
index 14bb8270..386b0556 100644
--- a/router/src/lib.rs
+++ b/router/src/lib.rs
@@ -1157,6 +1157,12 @@ pub(crate) struct GenerateResponse {
     pub details: Option<Details>,
 }
 
+#[derive(Serialize, ToSchema)]
+pub(crate) struct ChatTokenizeResponse {
+    pub(crate) tokenize_response: TokenizeResponse,
+    pub(crate) templated_text: String,
+}
+
 #[derive(Serialize, ToSchema)]
 #[serde(transparent)]
 pub(crate) struct TokenizeResponse(Vec<SimpleToken>);
diff --git a/router/src/server.rs b/router/src/server.rs
index dcbaa2ad..7655182a 100644
--- a/router/src/server.rs
+++ b/router/src/server.rs
@@ -8,6 +8,7 @@ use crate::kserve::{
     kserve_model_metadata, kserve_model_metadata_ready,
 };
 use crate::validation::ValidationError;
+use crate::ChatTokenizeResponse;
 use crate::{
     usage_stats, BestOfSequence, Details, ErrorResponse, FinishReason, FunctionName,
     GenerateParameters, GenerateRequest, GenerateResponse, GrammarType, HubModelInfo,
@@ -22,7 +23,7 @@ use crate::{
     CompletionRequest, CompletionType, DeltaToolCall, Function, Prompt, Tool, VertexRequest,
     VertexResponse,
 };
-use crate::{FunctionDefinition, HubPreprocessorConfig, ToolCall, ToolChoice, ToolType};
+use crate::{FunctionDefinition, HubPreprocessorConfig, ToolCall, ToolChoice, ToolType, Tools};
 use async_stream::__private::AsyncStream;
 use axum::extract::Extension;
 use axum::http::{HeaderMap, HeaderValue, Method, StatusCode};
@@ -115,6 +116,107 @@ async fn get_model_info(info: Extension<Info>) -> Json<Info> {
     Json(info.0)
 }
 
+#[utoipa::path(
+    post,
+    tag = "Text Generation Inference",
+    path = "/chat_tokenize",
+    request_body = ChatRequest,
+    responses((status = 200, description = "Templated and tokenized ChatRequest", body = ChatTokenizeResponse))
+)]
+async fn get_chat_tokenize(
+    Extension(infer): Extension<Infer>,
+    Json(req): Json<ChatRequest>,
+) -> Result<(HeaderMap, Json<ChatTokenizeResponse>), (StatusCode, Json<ErrorResponse>)> {
+    metrics::counter!("tgi_request_count").increment(1);
+
+    let ChatRequest {
+        model,
+        max_tokens,
+        messages,
+        seed,
+        stop,
+        stream,
+        tools,
+        tool_choice,
+        tool_prompt,
+        temperature,
+        response_format,
+        ..
+    } = req;
+
+    let tool_prompt = tool_prompt.unwrap_or_default();
+    let (inputs, _grammar, _tool_grammar) = prepare_chat_input(
+        &infer,
+        response_format,
+        tools,
+        tool_choice,
+        &tool_prompt,
+        messages,
+    )?;
+
+    let generate_request = GenerateRequest {
+        inputs,
+        parameters: GenerateParameters {
+            best_of: None,
+            temperature,
+            repetition_penalty: None,
+            frequency_penalty: None,
+            top_k: None,
+            top_p: None,
+            typical_p: None,
+            do_sample: true,
+            max_new_tokens: max_tokens,
+            return_full_text: None,
+            stop: stop.unwrap_or_default(),
+            truncate: None,
+            watermark: false,
+            details: false,
+            decoder_input_details: !stream,
+            seed,
+            top_n_tokens: None,
+            grammar: _grammar,
+            adapter_id: model.as_ref().filter(|m| *m != "tgi").map(String::from),
+        },
+    };
+
+    let input = generate_request.inputs.clone();
+    let encoding = infer.tokenize(generate_request).await?;
+    if let Some(encoding) = encoding {
+        let tokens: Vec<SimpleToken> = encoding
+            .get_ids()
+            .iter()
+            .zip(encoding.get_offsets())
+            .map(|(&id, &(start, stop))| {
+                let text = input
+                    .chars()
+                    .skip(start)
+                    .take(stop - start)
+                    .collect::<String>();
+                SimpleToken {
+                    id,
+                    text,
+                    start,
+                    stop,
+                }
+            })
+            .collect();
+
+        let resp = ChatTokenizeResponse {
+            tokenize_response: TokenizeResponse(tokens),
+            templated_text: input,
+        };
+        Ok((HeaderMap::new(), Json(resp)))
+    } else {
+        Err((
+            StatusCode::NOT_FOUND,
+            Json(ErrorResponse {
+                error: "No fast tokenizer or tokenizer.json for this model".to_string(),
+                error_type: "no fast tokenizer".to_string(),
+            }),
+        ))
+    }
+}
+
 #[utoipa::path(
 get,
 tag = "Text Generation Inference",
@@ -1034,63 +1136,14 @@ async fn chat_completions(
         Some(temperature) if temperature == 0.0 => (false, None),
         other => (true, other),
     };
-
-    // response_format and tools are mutually exclusive
-    if response_format.is_some() && tools.as_ref().is_some() {
-        metrics::counter!("tgi_request_failure", "err" => "validation").increment(1);
-        return Err((
-            StatusCode::UNPROCESSABLE_ENTITY,
-            Json(ErrorResponse {
-                error: "Grammar and tools are mutually exclusive".to_string(),
-                error_type: "grammar and tools".to_string(),
-            }),
-        ));
-    }
-
-    // extract tool grammar if present
-    let tool_grammar = match ToolGrammar::apply(tools, tool_choice) {
-        Ok(grammar) => grammar,
-        Err(err) => {
-            metrics::counter!("tgi_request_failure", "err" => "validation").increment(1);
-            tracing::error!("{err}");
-            return Err((
-                StatusCode::UNPROCESSABLE_ENTITY,
-                Json(ErrorResponse {
-                    error: err.to_string(),
-                    error_type: err.error_type().to_string(),
-                }),
-            ));
-        }
-    };
-
-    // determine the appropriate arguments for apply_chat_template
-    let tools_grammar_prompt = tool_grammar
-        .as_ref()
-        .map(|t| (GrammarType::Json(serde_json::json!(t)), tool_prompt));
-
-    let (tools_grammar_prompt, grammar) = match response_format {
-        Some(response_format) => (None, Some(response_format)),
-        None => (
-            tools_grammar_prompt.clone(),
-            tools_grammar_prompt.map(|(grammar, _)| grammar.clone()),
-        ),
-    };
-
-    // apply chat template to flatten the request into a single input
-    let inputs = match infer.apply_chat_template(messages, tools_grammar_prompt) {
-        Ok(inputs) => inputs,
-        Err(err) => {
-            metrics::counter!("tgi_request_failure", "err" => "validation").increment(1);
-            tracing::error!("{err}");
-            return Err((
-                StatusCode::UNPROCESSABLE_ENTITY,
-                Json(ErrorResponse {
-                    error: err.to_string(),
-                    error_type: err.error_type().to_string(),
-                }),
-            ));
-        }
-    };
+    let (inputs, grammar, tool_grammar) = prepare_chat_input(
+        &infer,
+        response_format,
+        tools,
+        tool_choice,
+        &tool_prompt,
+        messages,
+    )?;
 
     // build the request passing some parameters
     let generate_request = GenerateRequest {
@@ -1360,8 +1413,11 @@ async fn tokenize(
             .iter()
             .zip(encoding.get_offsets())
             .map(|(&id, &(start, stop))| {
-                let text: String =
-                    String::from_utf8_lossy(&input.as_bytes()[start..stop]).to_string();
+                let text = input
+                    .chars()
+                    .skip(start)
+                    .take(stop - start)
+                    .collect::<String>();
                 SimpleToken {
                     id,
                     text,
@@ -2036,6 +2092,7 @@ async fn start(
     }
     let info_routes = Router::new()
         .route("/", get(health))
+        .route("/chat_tokenize", post(get_chat_tokenize))
         .route("/info", get(get_model_info))
         .route("/health", get(health))
         .route("/ping", get(health))
@@ -2332,3 +2389,36 @@ fn create_post_processor(
 
     Ok(post_processor)
 }
+
+type PreparedInput = (String, Option<GrammarType>, Option<Tools>);
+
+fn prepare_chat_input(
+    infer: &Infer,
+    response_format: Option<GrammarType>,
+    tools: Option<Vec<Tool>>,
+    tool_choice: ToolChoice,
+    tool_prompt: &str,
+    messages: Vec<Message>,
+) -> Result<PreparedInput, InferError> {
+    if response_format.is_some() && tools.is_some() {
+        return Err(InferError::ToolError(
+            "Grammar and tools are mutually exclusive".into(),
+        ));
+    }
+
+    if let Some(format) = response_format {
+        let inputs = infer.apply_chat_template(messages, None)?;
+        return Ok((inputs, Some(format), None));
+    }
+
+    // if tools are set, apply the tool grammar and then the chat template
+    let tool_grammar: Option<Tools> = ToolGrammar::apply(tools, tool_choice)?;
+    let grammar = tool_grammar
+        .as_ref()
+        .map(|t| GrammarType::Json(serde_json::json!(t)));
+    let tools_grammar_prompt = tool_grammar
+        .as_ref()
+        .map(|t| (GrammarType::Json(serde_json::json!(t)), tool_prompt.into()));
+    let inputs = infer.apply_chat_template(messages, tools_grammar_prompt)?;
+    Ok((inputs, grammar, tool_grammar))
+}

From f8a5b381fe9e755be476d0ab5b20826766ba40cc Mon Sep 17 00:00:00 2001
From: drbh <david.richard.holtz@gmail.com>
Date: Tue, 6 Aug 2024 13:09:50 -0400
Subject: [PATCH 05/72] feat: prefer stop over eos_token to align with openai
 finish_reason (#2344)

---
 router/src/lib.rs    | 11 ++++++++++-
 router/src/server.rs |  4 ++--
 2 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/router/src/lib.rs b/router/src/lib.rs
index 386b0556..a956b058 100644
--- a/router/src/lib.rs
+++ b/router/src/lib.rs
@@ -619,7 +619,7 @@ impl ChatCompletion {
                 message,
                 logprobs: return_logprobs
                     .then(|| ChatCompletionLogprobs::from((details.tokens, details.top_tokens))),
-                finish_reason: details.finish_reason.to_string(),
+                finish_reason: details.finish_reason.format(true),
             }],
             usage: Usage {
                 prompt_tokens: details.prefill.len() as u32,
@@ -1117,6 +1117,15 @@ impl std::fmt::Display for FinishReason {
     }
 }
 
+impl FinishReason {
+    pub fn format(&self, use_stop: bool) -> String {
+        match self {
+            FinishReason::EndOfSequenceToken if use_stop => "stop".to_string(),
+            _ => self.to_string(),
+        }
+    }
+}
+
 #[derive(Serialize, ToSchema)]
 pub(crate) struct BestOfSequence {
     #[schema(example = "test")]
diff --git a/router/src/server.rs b/router/src/server.rs
index 7655182a..4b6fe50c 100644
--- a/router/src/server.rs
+++ b/router/src/server.rs
@@ -1021,7 +1021,7 @@ async fn completions(
                 total_tokens += details.prefill.len() as u32 + details.generated_tokens;
 
                 Ok(CompletionComplete {
-                    finish_reason: details.finish_reason.to_string(),
+                    finish_reason: details.finish_reason.format(true),
                     index: index as u32,
                     logprobs: None,
                     text: generation.generated_text,
@@ -1212,7 +1212,7 @@ async fn chat_completions(
                         tool_calls,
                         current_time,
                         logprobs,
-                        stream_token.details.map(|d| d.finish_reason.to_string()),
+                        stream_token.details.map(|d| d.finish_reason.format(true)),
                     ),
                 ))
                 .unwrap_or_else(|e| {

From 1768c00b9f124fcf92f513165da5228f355f3ea1 Mon Sep 17 00:00:00 2001
From: drbh <david.richard.holtz@gmail.com>
Date: Tue, 6 Aug 2024 13:10:19 -0400
Subject: [PATCH 06/72] feat: return the generated text when parsing fails
 (#2353)

---
 router/src/server.rs | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/router/src/server.rs b/router/src/server.rs
index 4b6fe50c..1d1cd36a 100644
--- a/router/src/server.rs
+++ b/router/src/server.rs
@@ -1246,9 +1246,13 @@ async fn chat_completions(
             .as_secs();
 
         let (tool_calls, output) = if tool_grammar.is_some() {
-            let gen_text_value: Value = serde_json::from_str(&generation.generated_text)
-                .map_err(|e| InferError::ToolError(e.to_string()))?;
-
+            let gen_text_value: Value =
+                serde_json::from_str(&generation.generated_text).map_err(|e| {
+                    InferError::ToolError(format!(
+                        "Failed to parse generated text: {} {:?}",
+                        e, generation.generated_text
+                    ))
+                })?;
             let function = gen_text_value.get("function").ok_or(InferError::ToolError(
                 "No function found in generated text".to_string(),
             ))?;

From a64d407d64de1ba168523df5391863b1f85c0824 Mon Sep 17 00:00:00 2001
From: drbh <david.richard.holtz@gmail.com>
Date: Tue, 6 Aug 2024 13:33:22 -0400
Subject: [PATCH 07/72] fix: default num_ln_in_parallel_attn to one if not
 supplied (#2364)

---
 .../models/custom_modeling/flash_rw_modeling.py               | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py b/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py
index 708641e7..0691da9b 100644
--- a/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py
@@ -473,7 +473,9 @@ class FlashRWLayer(nn.Module):
 class FlashRWLayerNorm(nn.Module):
     def __init__(self, config, prefix: str, weights):
         super().__init__()
-        self.num_ln = config.num_ln_in_parallel_attn
+        # Falcon2 includes the number of layer norms in the config
+        # in the case no number of layer norms is provided, we default to 1
+        self.num_ln = getattr(config, "num_ln_in_parallel_attn", 1)
 
         if self.num_ln == 1:
             self.input_ln = FastLayerNorm.load(

From 133015f40821706b1eaf9943aa3c9aa477d0c614 Mon Sep 17 00:00:00 2001
From: drbh <david.richard.holtz@gmail.com>
Date: Tue, 6 Aug 2024 15:25:30 -0400
Subject: [PATCH 08/72] fix: prefer original layernorm names for 180B (#2365)

---
 .../models/custom_modeling/flash_rw_modeling.py       | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py b/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py
index 0691da9b..fc002082 100644
--- a/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py
@@ -382,8 +382,13 @@ class FlashRWLayer(nn.Module):
 
         prefix = f"{prefix}.h.{layer_id}"
 
+        # NOTE: Falcon 180B uses the ln_attn prefix
+        ln_prefix = "input_layernorm"
+        if config.num_hidden_layers == 80:
+            ln_prefix = "ln_attn"
+
         self.input_layernorm = FastLayerNorm.load(
-            prefix=f"{prefix}.input_layernorm",
+            prefix=f"{prefix}.{ln_prefix}",
             weights=weights,
             eps=config.layer_norm_epsilon,
         )
@@ -477,6 +482,10 @@ class FlashRWLayerNorm(nn.Module):
         # in the case no number of layer norms is provided, we default to 1
         self.num_ln = getattr(config, "num_ln_in_parallel_attn", 1)
 
+        # Falcon 180B uses the ln_attn prefix and has 2 layer norms
+        if config.num_hidden_layers == 80:
+            self.num_ln = 2
+
         if self.num_ln == 1:
             self.input_ln = FastLayerNorm.load(
                 prefix=f"{prefix}.input_layernorm",

From 8094ecfc9ef22c838fa7d49db4af8301539619e3 Mon Sep 17 00:00:00 2001
From: almersawi <43927639+almersawi@users.noreply.github.com>
Date: Thu, 8 Aug 2024 03:45:23 +0400
Subject: [PATCH 09/72] fix: fix num_ln_in_parallel_attn attribute name typo in
 RWConfig (#2350)

Co-authored-by: Islam Almersawi <islam.almersawi@openinnovation.ai>
---
 .../models/custom_modeling/flash_rw_modeling.py                 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py b/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py
index fc002082..10f995a3 100644
--- a/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py
@@ -94,7 +94,7 @@ class RWConfig(PretrainedConfig):
             else kwargs.pop("n_head", 8)
         )
         self.layer_norm_epsilon = layer_norm_epsilon
-        self.num_ln_in_parallel_attention = num_ln_in_prallel_attention
+        self.num_ln_in_parallel_attn = num_ln_in_prallel_attention
         self.initializer_range = initializer_range
         self.use_cache = use_cache
         self.hidden_dropout = hidden_dropout

From 21267f3ca3f121302b86c1702cc2da6091164c55 Mon Sep 17 00:00:00 2001
From: drbh <david.richard.holtz@gmail.com>
Date: Wed, 7 Aug 2024 21:32:37 -0400
Subject: [PATCH 10/72] add gptj modeling in TGI #2366 (CI RUN) (#2372)

* add gptj modeling

Signed-off-by: Wang, Yi A <yi.a.wang@intel.com>

* fix: update docs for model addition

* fix: adjust syntax typo

* fix: adjust syntax typo again

---------

Signed-off-by: Wang, Yi A <yi.a.wang@intel.com>
Co-authored-by: Wang, Yi A <yi.a.wang@intel.com>
---
 docs/source/supported_models.md               |   1 +
 router/src/config.rs                          |   1 +
 .../text_generation_server/models/__init__.py |  43 ++
 .../custom_modeling/flash_gptj_modeling.py    | 405 ++++++++++++++++++
 4 files changed, 450 insertions(+)
 create mode 100644 server/text_generation_server/models/custom_modeling/flash_gptj_modeling.py

diff --git a/docs/source/supported_models.md b/docs/source/supported_models.md
index bc124f31..b78104df 100644
--- a/docs/source/supported_models.md
+++ b/docs/source/supported_models.md
@@ -32,6 +32,7 @@ Text Generation Inference enables serving optimized models on specific hardware
 - [Mpt](https://huggingface.co/mosaicml/mpt-7b-instruct)
 - [Gpt2](https://huggingface.co/openai-community/gpt2)
 - [Gpt Neox](https://huggingface.co/EleutherAI/gpt-neox-20b)
+- [Gptj](https://huggingface.co/EleutherAI/gpt-j-6b)
 - [Idefics](https://huggingface.co/HuggingFaceM4/idefics-9b) (Multimodal)
 
 
diff --git a/router/src/config.rs b/router/src/config.rs
index 7737165e..5d0be9c8 100644
--- a/router/src/config.rs
+++ b/router/src/config.rs
@@ -153,6 +153,7 @@ pub enum Config {
     Bloom,
     Mpt,
     Gpt2,
+    Gptj,
     GptNeox,
     Phi,
     #[serde(rename = "phi-msft")]
diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py
index ae791ef8..1f9c7526 100644
--- a/server/text_generation_server/models/__init__.py
+++ b/server/text_generation_server/models/__init__.py
@@ -132,6 +132,9 @@ try:
     from text_generation_server.models.custom_modeling.flash_gpt2_modeling import (
         FlashGPT2ForCausalLM,
     )
+    from text_generation_server.models.custom_modeling.flash_gptj_modeling import (
+        FlashGPTJForCausalLM,
+    )
     from text_generation_server.models.custom_modeling.idefics2 import (
         Idefics2ForConditionalGeneration,
     )
@@ -294,6 +297,11 @@ class ModelType(enum.Enum):
         "name": "Gpt Neox",
         "url": "https://huggingface.co/EleutherAI/gpt-neox-20b",
     }
+    GPTJ = {
+        "type": "gptj",
+        "name": "Gptj",
+        "url": "https://huggingface.co/EleutherAI/gpt-j-6b",
+    }
     IDEFICS = {
         "type": "idefics",
         "name": "Idefics",
@@ -641,6 +649,41 @@ def get_model(
                 dtype=dtype,
                 trust_remote_code=trust_remote_code,
             )
+    elif model_type == GPTJ:
+        if FLASH_ATTENTION:
+            try:
+                return FlashCausalLM(
+                    model_id=model_id,
+                    model_class=FlashGPTJForCausalLM,
+                    revision=revision,
+                    quantize=quantize,
+                    speculator=speculator,
+                    dtype=dtype,
+                    trust_remote_code=trust_remote_code,
+                    lora_adapter_ids=lora_adapter_ids,
+                )
+            except RuntimeError as e:
+                # Lots of legacy models with various weight names.
+                log_master(logger.warning, f"Couldn't load flash gptj variant: {e}")
+                return CausalLM.fallback(
+                    model_id,
+                    revision,
+                    quantize=quantize,
+                    speculator=speculator,
+                    dtype=dtype,
+                    trust_remote_code=trust_remote_code,
+                )
+        elif sharded:
+            raise NotImplementedError(FLASH_ATT_ERROR_MESSAGE.format("Sharded GPT-J"))
+        else:
+            return CausalLM.fallback(
+                model_id,
+                revision,
+                quantize=quantize,
+                speculator=speculator,
+                dtype=dtype,
+                trust_remote_code=trust_remote_code,
+            )
     elif model_type == GPT_NEOX:
         if FLASH_ATTENTION:
             from text_generation_server.models.custom_modeling.flash_neox_modeling import (
diff --git a/server/text_generation_server/models/custom_modeling/flash_gptj_modeling.py b/server/text_generation_server/models/custom_modeling/flash_gptj_modeling.py
new file mode 100644
index 00000000..eb667384
--- /dev/null
+++ b/server/text_generation_server/models/custom_modeling/flash_gptj_modeling.py
@@ -0,0 +1,405 @@
+# coding=utf-8
+# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
+#
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
+# and OPT implementations in this library. It has been modified from its
+# original forms to accommodate minor architectural differences compared
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import torch
+import torch.distributed
+
+from torch import nn
+from transformers.activations import ACT2FN
+from typing import Optional, List, Tuple
+
+from text_generation_server.layers.attention import (
+    paged_attention,
+    attention,
+    reshape_and_cache,
+)
+from text_generation_server.layers import (
+    TensorParallelRowLinear,
+    TensorParallelColumnLinear,
+    TensorParallelEmbedding,
+    SpeculativeHead,
+    get_linear,
+)
+from text_generation_server.layers.rotary import (
+    PositionRotaryEmbedding,
+)
+from text_generation_server.layers.layernorm import (
+    FastLayerNorm,
+)
+from text_generation_server.utils.import_utils import SYSTEM
+
+
+def load_attention(config, prefix: str, weights):
+    return TensorParallelColumnLinear.load_multi(
+        config,
+        prefixes=[f"{prefix}.q_proj", f"{prefix}.k_proj", f"{prefix}.v_proj"],
+        dim=0,
+        weights=weights,
+        bias=False,
+    )
+
+
+def load_row(config, prefix: str, weights, bias: bool):
+    weight = weights.get_weights_row(prefix)
+
+    if bias and weights.process_group.rank() == 0:
+        # Rank is only on the first rank process
+        bias = weights.get_tensor(f"{prefix}.bias")
+    else:
+        bias = None
+
+    linear = get_linear(weight, bias)
+    return TensorParallelRowLinear(linear, process_group=weights.process_group)
+
+
+class GPTJRotary(PositionRotaryEmbedding):
+    def forward(
+        self,
+        query: torch.Tensor,
+        key: torch.Tensor,
+        cos: torch.Tensor,
+        sin: torch.Tensor,
+    ):
+        # Such controlflows may add some overhead.
+        if SYSTEM == "cuda":
+            import rotary_emb
+
+            q1 = query[..., ::2]
+            q2 = query[..., 1::2]
+
+            rotary_emb.apply_rotary(q1, q2, cos, sin, q1, q2, False)
+
+            k1 = key[..., ::2]
+            k2 = key[..., 1::2]
+
+            rotary_emb.apply_rotary(k1, k2, cos, sin, k1, k2, False)
+        elif SYSTEM == "rocm":
+            from vllm._C import ops
+
+            # NOTE: On RoCm systems, we use a ROPE implementatation adapted from VLLM which launches a single kernel for both query/key, contrary to flash-attn implementation used on NVIDIA systems.
+            # Compiling flash-attn rotary on RoCm, it appears hipcc is unable to unroll loops, resulting in an even slower inference compared to eager: https://github.com/pytorch/pytorch/issues/113773
+
+            head_size = query.shape[-1]
+
+            # Inplace operation, updating query and key.
+            ops.rotary_embedding(query, key, head_size, cos, sin, False)
+        elif SYSTEM == "ipex":
+            import intel_extension_for_pytorch as ipex
+
+            ipex.llm.functional.rotary_embedding(
+                query, key, sin, cos, query.size(-1), False
+            )
+        else:
+            raise ValueError(
+                "Your system seem to be not supported. Please check your install or open an issue at https://github.com/huggingface/text-generation-inference/issues with a clear reproduction."
+            )
+
+
+class FlashGPTJAttention(torch.nn.Module):
+    def __init__(
+        self,
+        prefix: str,
+        config,
+        weights,
+    ):
+        super().__init__()
+        self.num_heads = config.num_attention_heads
+        self.hidden_size = config.hidden_size
+
+        self.head_size = self.hidden_size // self.num_heads
+        self.softmax_scale = self.head_size**-0.5
+        self.rotary_dim = config.rotary_dim
+
+        if self.num_heads % weights.process_group.size() != 0:
+            raise ValueError(
+                f"`num_heads` must be divisible by `num_shards` (got `num_heads`: {self.num_heads} "
+                f"and `num_shards`: {weights.process_group.size()}"
+            )
+        self.num_heads = self.num_heads // weights.process_group.size()
+
+        self.query_key_value = load_attention(
+            config,
+            prefix=prefix,
+            weights=weights,
+        )
+
+        self.o_proj = load_row(
+            config,
+            prefix=f"{prefix}.out_proj",
+            weights=weights,
+            bias=False,
+        )
+
+        self.kv_head_mapping = torch.arange(
+            0, self.num_heads, dtype=torch.int32, device=weights.device
+        )
+
+        self.rotary_emb = GPTJRotary.static(
+            config=config,
+            dim=self.rotary_dim,
+            base=10000,
+            device=weights.device,
+        )
+
+    def forward(
+        self,
+        hidden_states,
+        cos,
+        sin,
+        cu_seqlen_prefill,
+        kv_cache,
+        block_tables,
+        slots,
+        input_lengths,
+        max_s,
+    ):
+        query, key, value = self.query_key_value(hidden_states).split(
+            self.head_size * self.num_heads, dim=1
+        )
+        query = query.view(-1, self.num_heads, self.head_size)
+        key = key.view(-1, self.num_heads, self.head_size)
+        value = value.view(-1, self.num_heads, self.head_size)
+
+        # Compute rotary embeddings on rotary_ndims
+        if self.rotary_dim is not None:
+            self.rotary_emb(
+                query[..., : self.rotary_dim], key[..., : self.rotary_dim], cos, sin
+            )
+        else:
+            self.rotary_emb(query, key, cos, sin)
+
+        reshape_and_cache(key, value, kv_cache[0], kv_cache[1], slots)
+
+        # Prefill
+        if cu_seqlen_prefill is not None:
+            # flash attention
+            attn_output = attention(
+                query,
+                key,
+                value,
+                cu_seqlen_prefill,
+                max_s,
+                self.softmax_scale,
+            )
+        # Decode
+        else:
+            attn_output = paged_attention(
+                query,
+                kv_cache[0],
+                kv_cache[1],
+                self.kv_head_mapping,
+                self.softmax_scale,
+                block_tables,
+                input_lengths,
+                max_s,
+            )
+
+        return self.o_proj(attn_output.view(-1, self.num_heads * self.head_size))
+
+
+class GPTJMLP(nn.Module):
+    def __init__(self, prefix: str, config, weights):
+        super().__init__()
+        act = config.activation_function
+        self.act = (
+            ACT2FN[act]
+            if "gelu" not in act
+            else lambda x: torch.nn.functional.gelu(
+                x,
+                approximate=(
+                    "tanh" if act in ["gelu_fast", "gelu_pytorch_tanh"] else "none"
+                ),
+            )
+        )
+
+        self.fc_in = TensorParallelColumnLinear.load(
+            config, prefix=f"{prefix}.fc_in", weights=weights, bias=True
+        )
+
+        self.fc_out = load_row(
+            config,
+            prefix=f"{prefix}.fc_out",
+            weights=weights,
+            bias=True,
+        )
+
+    def forward(self, hidden_states):
+        hidden_states = self.fc_in(hidden_states)
+        hidden_states = self.act(hidden_states)
+        return self.fc_out(hidden_states)
+
+
+class FlashGPTJLayer(nn.Module):
+    def __init__(self, prefix: str, config, weights):
+        super().__init__()
+        self.self_attn = FlashGPTJAttention(
+            prefix=f"{prefix}.attn", config=config, weights=weights
+        )
+        self.mlp = GPTJMLP(prefix=f"{prefix}.mlp", config=config, weights=weights)
+
+        self.input_layernorm = FastLayerNorm.load(
+            prefix=f"{prefix}.ln_1", weights=weights, eps=config.layer_norm_epsilon
+        )
+
+    def forward(
+        self,
+        hidden_states,
+        residual,
+        cos,
+        sin,
+        cu_seqlen_prefill,
+        kv_cache,
+        block_tables,
+        slots,
+        input_lengths,
+        max_s,
+    ):
+        hidden_states, residual = self.input_layernorm(hidden_states, residual)
+        # Self Attention
+        attn_output = self.self_attn(
+            hidden_states,
+            cos,
+            sin,
+            cu_seqlen_prefill,
+            kv_cache,
+            block_tables,
+            slots,
+            input_lengths,
+            max_s,
+        )
+
+        feed_forward_hidden_states = self.mlp(hidden_states)
+
+        return attn_output + feed_forward_hidden_states, residual
+
+
+class FlashGPTJModel(torch.nn.Module):
+    def __init__(self, prefix: str, config, weights):
+        super().__init__()
+        self.config = config
+
+        self.wte = TensorParallelEmbedding(prefix=f"{prefix}.wte", weights=weights)
+        self.layers = nn.ModuleList(
+            [
+                FlashGPTJLayer(
+                    prefix=(
+                        f"h.{layer_id}" if not prefix else f"{prefix}.h.{layer_id}"
+                    ),
+                    config=config,
+                    weights=weights,
+                )
+                for layer_id in range(config.num_hidden_layers)
+            ]
+        )
+
+        self.ln_f = FastLayerNorm.load(
+            prefix="ln_f" if not prefix else f"{prefix}.ln_f",
+            weights=weights,
+            eps=config.layer_norm_epsilon,
+        )
+
+        self.gradient_checkpointing = False
+
+        self.head_size = self.layers[0].self_attn.head_size
+        self.num_heads = self.layers[0].self_attn.num_heads
+
+    def forward(
+        self,
+        input_ids: Optional[torch.LongTensor],
+        position_ids: torch.Tensor,
+        cu_seqlen_prefill: Optional[torch.Tensor],
+        kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
+        block_tables: torch.Tensor,
+        slots: torch.Tensor,
+        input_lengths: torch.Tensor,
+        max_s: int,
+        prefill_cache_indices: Optional[torch.Tensor],
+    ) -> torch.Tensor:
+        hidden_states = self.wte(input_ids)
+
+        # Get rotary cos and sin for this forward
+        # Avoid to index in each layer
+        cos, sin = self.layers[0].self_attn.rotary_emb.get_cos_sin(
+            position_ids, max_s, hidden_states.dtype
+        )
+
+        residual = None
+        for i, layer in enumerate(self.layers):
+            hidden_states, residual = layer(
+                hidden_states,
+                residual,
+                cos,
+                sin,
+                cu_seqlen_prefill,
+                kv_cache[i],
+                block_tables,
+                slots,
+                input_lengths,
+                max_s,
+            )
+
+        hidden_states, _ = self.ln_f(hidden_states, residual)
+
+        return hidden_states
+
+
+class FlashGPTJForCausalLM(torch.nn.Module):
+    def __init__(self, prefix: str, config, weights):
+        super().__init__()
+        if not prefix:
+            prefix = "transformer"
+        else:
+            prefix = f"{prefix}.transformer"
+        self.model = FlashGPTJModel(prefix, config, weights)
+        self.lm_head = SpeculativeHead.load(
+            config,
+            prefix="lm_head",
+            weights=weights,
+        )
+
+    def forward(
+        self,
+        input_ids: torch.Tensor,
+        position_ids: torch.Tensor,
+        cu_seqlen_prefill: Optional[torch.Tensor],
+        kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
+        block_tables: torch.Tensor,
+        slots: torch.Tensor,
+        input_lengths: torch.Tensor,
+        max_s: int,
+        prefill_cache_indices: Optional[torch.Tensor] = None,
+        lm_head_indices: Optional[torch.Tensor] = None,
+        adapter_data: Optional[torch.Tensor] = None,
+    ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
+        hidden_states = self.model(
+            input_ids,
+            position_ids,
+            cu_seqlen_prefill,
+            kv_cache,
+            block_tables,
+            slots,
+            input_lengths,
+            max_s,
+            prefill_cache_indices=prefill_cache_indices,
+        )
+        if lm_head_indices is not None:
+            hidden_states = hidden_states[lm_head_indices]
+        logits, speculative_logits = self.lm_head(hidden_states)
+        return logits, speculative_logits

From a379d5536bb2de55154dc09c3a1f24ce58cb7df5 Mon Sep 17 00:00:00 2001
From: drbh <david.richard.holtz@gmail.com>
Date: Wed, 7 Aug 2024 23:14:02 -0400
Subject: [PATCH 11/72] Fix the prefix for OPT model in opt_modelling.py #2370
 (CI RUN) (#2371)

* Fix the bug

* fix: run lints

* fix: small syntax tweak

---------

Co-authored-by: Sadra Barikbin <sadraqazvin1@yahoo.com>
---
 integration-tests/models/test_opt.py          | 19 ++++++++++++++
 .../models/custom_modeling/opt_modeling.py    | 25 ++++++++++---------
 2 files changed, 32 insertions(+), 12 deletions(-)
 create mode 100644 integration-tests/models/test_opt.py

diff --git a/integration-tests/models/test_opt.py b/integration-tests/models/test_opt.py
new file mode 100644
index 00000000..cbeb6376
--- /dev/null
+++ b/integration-tests/models/test_opt.py
@@ -0,0 +1,19 @@
+import pytest
+
+
+@pytest.fixture(scope="module")
+def opt_sharded_handle(launcher):
+    with launcher("facebook/opt-6.7b", num_shard=2) as handle:
+        yield handle
+
+
+@pytest.fixture(scope="module")
+async def opt_sharded(opt_sharded_handle):
+    await opt_sharded_handle.health(300)
+    return opt_sharded_handle.client
+
+
+@pytest.mark.release
+@pytest.mark.asyncio
+async def test_opt(opt_sharded):
+    pass
diff --git a/server/text_generation_server/models/custom_modeling/opt_modeling.py b/server/text_generation_server/models/custom_modeling/opt_modeling.py
index 84a1c069..bd440321 100644
--- a/server/text_generation_server/models/custom_modeling/opt_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/opt_modeling.py
@@ -98,7 +98,9 @@ class OPTLearnedPositionalEmbedding(nn.Module):
         super().__init__()
         self.offset = 2
         self.weight = nn.Parameter(
-            weights.get_tensor(f"{prefix}.decoder.embed_positions.weight")
+            weights.get_tensor(
+                f"{prefix + '.' if prefix else ''}decoder.embed_positions.weight"
+            )
         )
 
     def forward(
@@ -315,7 +317,7 @@ class OPTDecoderLayer(nn.Module):
         super().__init__()
         self.process_group = weights.process_group
         self.hidden_size = config.hidden_size
-        prefix = f"{prefix}.decoder.layers.{layer_id}"
+        prefix = f"{prefix + '.' if prefix else ''}decoder.layers.{layer_id}"
         self.self_attn = OPTAttention(
             config,
             prefix=f"{prefix}.self_attn",
@@ -437,15 +439,17 @@ class OPTDecoder(OPTPreTrainedModel):
         self.max_target_positions = config.max_position_embeddings
         self.vocab_size = config.vocab_size
 
+        prefix = prefix + "." if prefix else ""
+
         self.embed_tokens = TensorParallelEmbedding(
-            prefix=f"{prefix}.decoder.embed_tokens", weights=weights
+            prefix=f"{prefix}decoder.embed_tokens", weights=weights
         )
         self.embed_positions = OPTLearnedPositionalEmbedding(prefix, weights)
 
         if config.word_embed_proj_dim != config.hidden_size:
             self.project_out = FastLinear.load(
                 config,
-                prefix=f"{prefix}.decoder.project_out",
+                prefix=f"{prefix}decoder.project_out",
                 weights=weights,
                 bias=False,
             )
@@ -455,7 +459,7 @@ class OPTDecoder(OPTPreTrainedModel):
         if config.word_embed_proj_dim != config.hidden_size:
             self.project_in = FastLinear.load(
                 config,
-                prefix=f"{prefix}.decoder.project_in",
+                prefix=f"{prefix}decoder.project_in",
                 weights=weights,
                 bias=False,
             )
@@ -467,7 +471,7 @@ class OPTDecoder(OPTPreTrainedModel):
         # see https://github.com/facebookresearch/metaseq/pull/164
         if config.do_layer_norm_before and not config._remove_final_layer_norm:
             self.final_layer_norm = nn.LayerNorm.load(
-                prefix=f"{prefix}.decoder.final_layer_norm", weights=weights, eps=EPS
+                prefix=f"{prefix}decoder.final_layer_norm", weights=weights, eps=EPS
             )
         else:
             self.final_layer_norm = None
@@ -752,15 +756,12 @@ class OPTForCausalLM(OPTPreTrainedModel):
     def __init__(self, prefix, config, weights):
         super().__init__(config)
 
-        if not prefix:
-            prefix = "model"
-        else:
-            prefix = f"{prefix}.model"
-
         self.model = OPTModel(prefix, config, weights)
 
         self.lm_head = SpeculativeHead.load(
-            config, prefix=f"{prefix}.decoder.embed_tokens", weights=weights
+            config,
+            prefix=f"{prefix + '.' if prefix else ''}decoder.embed_tokens",
+            weights=weights,
         )
 
     def forward(

From 82d19d7723c085a0f0bd37494ee2e2c41e1323ca Mon Sep 17 00:00:00 2001
From: drbh <david.richard.holtz@gmail.com>
Date: Thu, 8 Aug 2024 11:14:06 -0400
Subject: [PATCH 12/72] Pr 2374 ci branch (#2378)

* Update __init__.py

Fix issue with NoneType comparison for max_input_tokens and sliding_window

- Add default values for max_input_tokens and sliding_window to handle None cases.
- Ensure the comparison between max_input_tokens and sliding_window is handled correctly to prevent TypeError.
- This change addresses the error: TypeError: '<=' not supported between instances of 'int' and 'NoneType'.

* Update __init__.py

Handle NoneType in sliding_window comparison to fix TypeError in __init__.py by ensuring the comparison logic accounts for NoneType values, preventing errors and improving code robustness.

* fix: syntax/style tweak

---------

Co-authored-by: Praz <prazanth2006@gmail.com>
---
 server/text_generation_server/models/__init__.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py
index 1f9c7526..da14d083 100644
--- a/server/text_generation_server/models/__init__.py
+++ b/server/text_generation_server/models/__init__.py
@@ -490,7 +490,12 @@ def get_model(
         raise RuntimeError(
             "Sharding is currently not supported with `exl2` quantization"
         )
-    sliding_window = config_dict.get("sliding_window", -1)
+
+    sliding_window = (
+        config_dict.get("sliding_window")
+        if config_dict.get("sliding_window") is not None
+        else -1
+    )
 
     if max_input_tokens is not None and max_input_tokens <= sliding_window:
         sliding_window = -1

From 689b1abbf68cd929f41b72b06cc9e44b266fed53 Mon Sep 17 00:00:00 2001
From: "Wang, Yi" <yi.a.wang@intel.com>
Date: Fri, 9 Aug 2024 00:08:52 +0800
Subject: [PATCH 13/72] fix EleutherAI/gpt-neox-20b does not work in tgi
 (#2346)

Signed-off-by: Wang, Yi A <yi.a.wang@intel.com>
---
 .../models/custom_modeling/flash_neox_modeling.py      | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py b/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py
index b1b03ad7..67237d5c 100644
--- a/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py
@@ -153,8 +153,16 @@ class FlashNeoxAttention(torch.nn.Module):
         qkv = self.query_key_value(hidden_states)
         qkv = qkv.view(-1, 3, self.num_heads, self.head_size)
 
+        # Compute rotary embeddings on rotary_ndims
+        query_rot = qkv[:, 0][..., : self.rotary_dim]
+        query_pass = qkv[:, 0][..., self.rotary_dim :]
+        key_rot = qkv[:, 1][..., : self.rotary_dim]
+        key_pass = qkv[:, 1][..., self.rotary_dim :]
+
         # Inplace rotary
-        self.rotary_emb(qkv[:, 0], qkv[:, 1], cos, sin)
+        self.rotary_emb(query_rot, key_rot, cos, sin)
+        qkv[:, 0] = torch.cat((query_rot, query_pass), dim=-1)
+        qkv[:, 1] = torch.cat((key_rot, key_pass), dim=-1)
 
         reshape_and_cache(qkv[:, 1], qkv[:, 2], kv_cache[0], kv_cache[1], slots)
 

From 2ca598063443d0bf4ba3b1f0fb3dbc17e60f8a67 Mon Sep 17 00:00:00 2001
From: drbh <david.richard.holtz@gmail.com>
Date: Thu, 8 Aug 2024 12:30:29 -0400
Subject: [PATCH 14/72] Pr 2337 ci branch (#2379)

* hotfix: fix xpu crash brought by code refine. torch.xpu rely on import ipex

Signed-off-by: Wang, Yi A <yi.a.wang@intel.com>

* reable gemma2 in xpu

Signed-off-by: Wang, Yi A <yi.a.wang@intel.com>

* fix in regression in ipex flashattention

Signed-off-by: Wang, Yi A <yi.a.wang@intel.com>

---------

Signed-off-by: Wang, Yi A <yi.a.wang@intel.com>
Co-authored-by: Wang, Yi A <yi.a.wang@intel.com>
---
 server/text_generation_server/layers/attention/ipex.py | 7 ++++++-
 server/text_generation_server/utils/import_utils.py    | 2 ++
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/server/text_generation_server/layers/attention/ipex.py b/server/text_generation_server/layers/attention/ipex.py
index e0956b26..d7cf780a 100644
--- a/server/text_generation_server/layers/attention/ipex.py
+++ b/server/text_generation_server/layers/attention/ipex.py
@@ -2,6 +2,7 @@ import intel_extension_for_pytorch as ipex
 import torch
 from text_generation_server.models.flash_causal_lm import BLOCK_SIZE
 from text_generation_server.layers.attention import Seqlen
+from typing import Optional
 
 SUPPORTS_WINDOWING = False
 
@@ -15,11 +16,12 @@ def attention(
     softmax_scale,
     window_size_left=-1,
     causal=True,
+    softcap: Optional[float] = None,
 ):
     out = torch.empty_like(q)
 
     # We do not need to check window_size_left (not supported) here, so it is already checked ahead of time at model load.
-    return ipex.llm.functional.varlen_attention(
+    ipex.llm.functional.varlen_attention(
         q,
         k,
         v,
@@ -36,6 +38,8 @@ def attention(
         None,
     )
 
+    return out
+
 
 def reshape_and_cache(
     key: torch.Tensor,
@@ -58,6 +62,7 @@ def paged_attention(
     block_tables: torch.Tensor,
     seqlen: Seqlen,
     max_s: int,
+    softcap: Optional[float] = None,
 ):
     out = torch.empty_like(query)
     ipex.llm.modules.PagedAttention.single_query_cached_kv_attention(
diff --git a/server/text_generation_server/utils/import_utils.py b/server/text_generation_server/utils/import_utils.py
index 7c053014..782b4f15 100644
--- a/server/text_generation_server/utils/import_utils.py
+++ b/server/text_generation_server/utils/import_utils.py
@@ -56,6 +56,8 @@ elif torch.version.cuda is not None and torch.cuda.is_available():
     get_free_memory = get_cuda_free_memory
 elif is_ipex_available():
     SYSTEM = "ipex"
+    import intel_extension_for_pytorch  # noqa: F401
+
     if hasattr(torch, "xpu") and torch.xpu.is_available():
         empty_cache = torch.xpu.empty_cache
         synchronize = torch.xpu.synchronize

From f8521900601578a070ab5bb4275cbb2cd45b8e01 Mon Sep 17 00:00:00 2001
From: drbh <david.richard.holtz@gmail.com>
Date: Thu, 8 Aug 2024 14:08:56 -0400
Subject: [PATCH 15/72] fix: prefer hidden_activation over hidden_act in gemma2
 (#2381)

---
 .../models/custom_modeling/flash_gemma2_modeling.py             | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/server/text_generation_server/models/custom_modeling/flash_gemma2_modeling.py b/server/text_generation_server/models/custom_modeling/flash_gemma2_modeling.py
index de86f514..54d212e6 100644
--- a/server/text_generation_server/models/custom_modeling/flash_gemma2_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/flash_gemma2_modeling.py
@@ -265,7 +265,7 @@ class FlashGemma2Attention(torch.nn.Module):
 class Gemma2MLP(nn.Module):
     def __init__(self, prefix, config, weights):
         super().__init__()
-        act = config.hidden_act
+        act = config.hidden_activation
         self.act = (
             ACT2FN[act]
             if "gelu" not in act

From cb3ae30284ada6d15822a4ccde9156b8e93ef2b6 Mon Sep 17 00:00:00 2001
From: Vaibhav Srivastav <vaibhavs10@gmail.com>
Date: Thu, 8 Aug 2024 22:06:57 +0200
Subject: [PATCH 16/72] Update Quantization docs and minor doc fix. (#2368)

* Update Quantization docs and minor doc fix.

* update readme with latest quants info

* Apply suggestions from code review

Co-authored-by: Pedro Cuenca <pedro@huggingface.co>

* up

---------

Co-authored-by: Pedro Cuenca <pedro@huggingface.co>
---
 README.md                                     |  2 +
 docs/openapi.json                             |  2 +-
 .../source/basic_tutorials/preparing_model.md |  2 +-
 .../basic_tutorials/visual_language_models.md |  2 +-
 docs/source/conceptual/quantization.md        | 59 +++++++++++--------
 5 files changed, 41 insertions(+), 26 deletions(-)

diff --git a/README.md b/README.md
index a88e0437..cf7f1d22 100644
--- a/README.md
+++ b/README.md
@@ -48,6 +48,8 @@ Text Generation Inference (TGI) is a toolkit for deploying and serving Large Lan
   - [GPT-Q](https://arxiv.org/abs/2210.17323)
   - [EETQ](https://github.com/NetEase-FuXi/EETQ)
   - [AWQ](https://github.com/casper-hansen/AutoAWQ)
+  - [Marlin](https://github.com/IST-DASLab/marlin)
+  - [fp8]()
 - [Safetensors](https://github.com/huggingface/safetensors) weight loading
 - Watermarking with [A Watermark for Large Language Models](https://arxiv.org/abs/2301.10226)
 - Logits warper (temperature scaling, top-p, top-k, repetition penalty, more details see [transformers.LogitsProcessor](https://huggingface.co/docs/transformers/internal/generation_utils#transformers.LogitsProcessor))
diff --git a/docs/openapi.json b/docs/openapi.json
index ed9b0b96..9d281a48 100644
--- a/docs/openapi.json
+++ b/docs/openapi.json
@@ -2080,4 +2080,4 @@
       "description": "Hugging Face Text Generation Inference API"
     }
   ]
-}
+}
\ No newline at end of file
diff --git a/docs/source/basic_tutorials/preparing_model.md b/docs/source/basic_tutorials/preparing_model.md
index 71ca5598..456ade44 100644
--- a/docs/source/basic_tutorials/preparing_model.md
+++ b/docs/source/basic_tutorials/preparing_model.md
@@ -4,7 +4,7 @@ Text Generation Inference improves the model in several aspects.
 
 ## Quantization
 
-TGI supports [bits-and-bytes](https://github.com/TimDettmers/bitsandbytes#bitsandbytes), [GPT-Q](https://arxiv.org/abs/2210.17323) and [AWQ](https://arxiv.org/abs/2306.00978) quantization. To speed up inference with quantization, simply set `quantize` flag to `bitsandbytes`, `gptq` or `awq` depending on the quantization technique you wish to use. When using GPT-Q quantization, you need to point to one of the models [here](https://huggingface.co/models?search=gptq) when using AWQ quantization, you need to point to one of the models [here](https://huggingface.co/models?search=awq). To get more information about quantization, please refer to [quantization guide](./../conceptual/quantization)
+TGI supports [bits-and-bytes](https://github.com/TimDettmers/bitsandbytes#bitsandbytes), [GPT-Q](https://arxiv.org/abs/2210.17323), [AWQ](https://arxiv.org/abs/2306.00978), [Marlin](https://github.com/IST-DASLab/marlin), [EETQ](https://github.com/NetEase-FuXi/EETQ), [EXL2](https://github.com/turboderp/exllamav2), and [fp8](https://developer.nvidia.com/blog/nvidia-arm-and-intel-publish-fp8-specification-for-standardization-as-an-interchange-format-for-ai/) quantization. To speed up inference with quantization, simply set `quantize` flag to `bitsandbytes`, `gptq`, `awq`, `marlin`, `exl2`, `eetq` or `fp8` depending on the quantization technique you wish to use. When using GPT-Q quantization, you need to point to one of the models [here](https://huggingface.co/models?search=gptq). Similarly, when using AWQ quantization, you need to point to one of [these models](https://huggingface.co/models?search=awq). To get more information about quantization, please refer to [quantization guide](./../conceptual/quantization)
 
 
 ## RoPE Scaling
diff --git a/docs/source/basic_tutorials/visual_language_models.md b/docs/source/basic_tutorials/visual_language_models.md
index 3770db0b..f152a2f0 100644
--- a/docs/source/basic_tutorials/visual_language_models.md
+++ b/docs/source/basic_tutorials/visual_language_models.md
@@ -84,7 +84,7 @@ print(chat)
 
 ```
 
-or with OpenAi's library:
+or with OpenAI's [client library](https://github.com/openai/openai-python):
 
 ```python
 from openai import OpenAI
diff --git a/docs/source/conceptual/quantization.md b/docs/source/conceptual/quantization.md
index 8f26fdba..7507687f 100644
--- a/docs/source/conceptual/quantization.md
+++ b/docs/source/conceptual/quantization.md
@@ -1,6 +1,40 @@
 # Quantization
 
-TGI offers GPTQ and bits-and-bytes quantization to quantize large language models.
+TGI offers many quantization schemes to run LLMs effectively and fast based on your use-case. TGI supports GPTQ, AWQ, bits-and-bytes, EETQ, Marlin, EXL2 and fp8 quantization.
+
+To leverage GPTQ, AWQ, Marlin and EXL2 quants, you must provide pre-quantized weights. Whereas for bits-and-bytes, EETQ and fp8, weights are quantized by TGI on the fly.
+
+We recommend using the official quantization scripts for creating your quants:
+1. [AWQ](https://github.com/casper-hansen/AutoAWQ/blob/main/examples/quantize.py)
+2. [GPTQ/ Marlin](https://github.com/AutoGPTQ/AutoGPTQ/blob/main/examples/quantization/basic_usage.py)
+3. [EXL2](https://github.com/turboderp/exllamav2/blob/master/doc/convert.md)
+
+For on-the-fly quantization you simply need to pass one of the supported quantization types and TGI takes care of the rest. 
+
+## Quantization with bitsandbytes
+
+bitsandbytes is a library used to apply 8-bit and 4-bit quantization to models. Unlike GPTQ quantization, bitsandbytes doesn't require a calibration dataset or any post-processing – weights are automatically quantized on load. However, inference with bitsandbytes is slower than GPTQ or FP16 precision.
+
+8-bit quantization enables multi-billion parameter scale models to fit in smaller hardware without degrading performance too much.
+In TGI, you can use 8-bit quantization by adding `--quantize bitsandbytes` like below 👇
+
+```bash
+docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:latest --model-id $model --quantize bitsandbytes
+```
+
+4-bit quantization is also possible with bitsandbytes. You can choose one of the following 4-bit data types: 4-bit float (`fp4`), or 4-bit `NormalFloat` (`nf4`). These data types were introduced in the context of parameter-efficient fine-tuning, but you can apply them for inference by automatically converting the model weights on load.
+
+In TGI, you can use 4-bit quantization by adding `--quantize bitsandbytes-nf4` or `--quantize bitsandbytes-fp4` like below 👇
+
+```bash
+docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:latest --model-id $model --quantize bitsandbytes-nf4
+```
+
+You can get more information about 8-bit quantization by reading this [blog post](https://huggingface.co/blog/hf-bitsandbytes-integration), and 4-bit quantization by reading [this blog post](https://huggingface.co/blog/4bit-transformers-bitsandbytes).
+
+Use `eetq` or `fp8` for other quantization schemes.
+
+In addition to this, TGI allows creating GPTQ quants directly by passing the model weights and a calibration dataset.
 
 ## Quantization with GPTQ
 
@@ -35,25 +69,4 @@ text-generation-launcher --model-id /data/falcon-40b-gptq/ --sharded true --num-
 You can learn more about the quantization options by running `text-generation-server quantize --help`.
 
 If you wish to do more with GPTQ models (e.g. train an adapter on top), you can read about transformers GPTQ integration [here](https://huggingface.co/blog/gptq-integration).
-You can learn more about GPTQ from the [paper](https://arxiv.org/pdf/2210.17323.pdf).
-
-## Quantization with bitsandbytes
-
-bitsandbytes is a library used to apply 8-bit and 4-bit quantization to models. Unlike GPTQ quantization, bitsandbytes doesn't require a calibration dataset or any post-processing – weights are automatically quantized on load. However, inference with bitsandbytes is slower than GPTQ or FP16 precision.
-
-8-bit quantization enables multi-billion parameter scale models to fit in smaller hardware without degrading performance too much.
-In TGI, you can use 8-bit quantization by adding `--quantize bitsandbytes` like below 👇
-
-```bash
-docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:latest --model-id $model --quantize bitsandbytes
-```
-
-4-bit quantization is also possible with bitsandbytes. You can choose one of the following 4-bit data types: 4-bit float (`fp4`), or 4-bit `NormalFloat` (`nf4`). These data types were introduced in the context of parameter-efficient fine-tuning, but you can apply them for inference by automatically converting the model weights on load.
-
-In TGI, you can use 4-bit quantization by adding `--quantize bitsandbytes-nf4` or `--quantize bitsandbytes-fp4` like below 👇
-
-```bash
-docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:latest --model-id $model --quantize bitsandbytes-nf4
-```
-
-You can get more information about 8-bit quantization by reading this [blog post](https://huggingface.co/blog/hf-bitsandbytes-integration), and 4-bit quantization by reading [this blog post](https://huggingface.co/blog/4bit-transformers-bitsandbytes).
+You can learn more about GPTQ from the [paper](https://arxiv.org/pdf/2210.17323.pdf).
\ No newline at end of file

From 6d06473cf48e19e7382b27940f993a5f48c83997 Mon Sep 17 00:00:00 2001
From: drbh <david.richard.holtz@gmail.com>
Date: Fri, 9 Aug 2024 04:54:32 -0400
Subject: [PATCH 17/72] Pr 2352 ci branch (#2382)

* Fix unsigned integer underflow

Passing --max-batch-size to the launcher actually had no effect
because after a few requests the max_size passed to State::next_batch
would underflow becoming a largo positive number.

In the scheduler, as soon as the cached batch size reached the
max_batch_size the max_size passed to next_batch becomes 0.
Since the only check in that funcion is
```
if Some(batch_requests.len()) == max_size {
    break;
}
```
and it's called after the `batch_requests.len()` has
become 1, it doesn't do anything to prevent more than 0
requests from being batched.

Now we have cached batch in the server that is large than
max_batch_size and `max_size - batch_size as usize`
underflows.

Signed-off-by: Max de Bayser <mbayser@br.ibm.com>

* fix: update v3 scheduler and ensure max_batch_size > 0

---------

Signed-off-by: Max de Bayser <mbayser@br.ibm.com>
Co-authored-by: Max de Bayser <mbayser@br.ibm.com>
---
 backends/v3/src/backend.rs       | 3 ++-
 backends/v3/src/main.rs          | 8 ++++++++
 backends/v3/src/queue.rs         | 7 +++++++
 router/src/infer/v2/queue.rs     | 7 +++++++
 router/src/infer/v2/scheduler.rs | 4 ++--
 5 files changed, 26 insertions(+), 3 deletions(-)

diff --git a/backends/v3/src/backend.rs b/backends/v3/src/backend.rs
index d82355de..6b3e0526 100644
--- a/backends/v3/src/backend.rs
+++ b/backends/v3/src/backend.rs
@@ -168,7 +168,8 @@ pub(crate) async fn batching_task(
                 };
 
                 let token_budget = max_batch_total_tokens.saturating_sub(batch_max_tokens);
-                let max_size = max_batch_size.map(|max_size| max_size - batch_size as usize);
+                let max_size =
+                    max_batch_size.map(|max_size| max_size.saturating_sub(batch_size as usize));
 
                 // Try to get a new batch
                 if let Some((mut new_entries, new_batch, span)) = queue
diff --git a/backends/v3/src/main.rs b/backends/v3/src/main.rs
index 21952e66..471ddb5a 100644
--- a/backends/v3/src/main.rs
+++ b/backends/v3/src/main.rs
@@ -150,6 +150,14 @@ async fn main() -> Result<(), RouterError> {
         }
     }
 
+    if let Some(max_batch_size) = max_batch_size {
+        if max_batch_size == 0 {
+            return Err(RouterError::ArgumentValidation(
+                "`max_batch_size` must be > 0".to_string(),
+            ));
+        }
+    }
+
     let (backend, _backend_info) = connect_backend(
         max_input_tokens,
         max_total_tokens,
diff --git a/backends/v3/src/queue.rs b/backends/v3/src/queue.rs
index 9427bd60..b457389c 100644
--- a/backends/v3/src/queue.rs
+++ b/backends/v3/src/queue.rs
@@ -226,6 +226,13 @@ impl State {
             }
         }
 
+        if let Some(max_size) = max_size {
+            if max_size == 0 {
+                tracing::debug!("No capacity");
+                return None;
+            }
+        }
+
         // Pad prefill_token_budget to be a multiple of block size
         let prefill_token_budget =
             ((prefill_token_budget + self.block_size - 1) / self.block_size) * self.block_size;
diff --git a/router/src/infer/v2/queue.rs b/router/src/infer/v2/queue.rs
index 0b51645a..696cbfc8 100644
--- a/router/src/infer/v2/queue.rs
+++ b/router/src/infer/v2/queue.rs
@@ -205,6 +205,13 @@ impl State {
             }
         }
 
+        if let Some(max_size) = max_size {
+            if max_size == 0 {
+                tracing::debug!("No capacity");
+                return None;
+            }
+        }
+
         // Pad prefill_token_budget to be a multiple of block size
         let prefill_token_budget =
             ((prefill_token_budget + self.block_size - 1) / self.block_size) * self.block_size;
diff --git a/router/src/infer/v2/scheduler.rs b/router/src/infer/v2/scheduler.rs
index 3d6c36cf..cc333674 100644
--- a/router/src/infer/v2/scheduler.rs
+++ b/router/src/infer/v2/scheduler.rs
@@ -161,8 +161,8 @@ pub(crate) async fn batching_task(
                 };
 
                 let token_budget = max_batch_total_tokens.saturating_sub(batch_max_tokens);
-                let max_size = max_batch_size.map(|max_size| max_size - batch_size as usize);
-
+                let max_size =
+                    max_batch_size.map(|max_size| max_size.saturating_sub(batch_size as usize));
                 // Try to get a new batch
                 if let Some((mut new_entries, new_batch, span)) = queue
                     .next_batch(min_size, max_size, max_batch_prefill_tokens, token_budget)

From 7830de1566df365e6cb9ce0a955e8e2ac1b28ca6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Dani=C3=ABl=20de=20Kok?= <me@danieldk.eu>
Date: Fri, 9 Aug 2024 11:42:00 +0200
Subject: [PATCH 18/72] Add FlashInfer support (#2354)

This change adds support for FlashInfer. FlashInfer can be enabled using
`FLASH_INFER=1` and is currently only implemented in `FlashCausalLM`.
Since this functionality is currently only for testing, FlashInfer is
not installed anywhere yet.

The FlashInfer API is quite different from FlashAttention/vLLM in that
it requires more global bookkeeping:

* A wrapper class needs to be contstructed (which we just call *state*).
  Since this is fairly expensive (due to pinned host memory allocation),
  we only do this once in a FlashCausalLM instance or for each CUDA
  Graph size.
* Each model forward call needs to be wrapped in `begin_forward` and
  `end_forward`. This sets up data structures that can be reused for all
  calls to attention for that forward call.

When calling attention, we need access to the state object. To avoid
passing an argument down the call chain (which would require changes to
all models), we use a context variable.

Each model forward call is wrapped using a context manager that does all
the bookkeeping for such a call:

* Set the context variable to the forward call's state.
* Call `begin_forward` on the state.
* Yield.
* Call `end_forward` on the state.
* Reset the context variable.

We cannot use a single shared global variable for this, since e.g. CUDA
Graphs of different sizes each have their own state.
---
 .../layers/attention/common.py                |   4 +-
 .../layers/attention/cuda.py                  |  46 ++++-
 .../layers/attention/flash_infer.py           | 164 +++++++++++++++++
 .../models/flash_causal_lm.py                 | 171 ++++++++++++++----
 .../text_generation_server/models/globals.py  |   5 +
 5 files changed, 346 insertions(+), 44 deletions(-)
 create mode 100644 server/text_generation_server/layers/attention/flash_infer.py

diff --git a/server/text_generation_server/layers/attention/common.py b/server/text_generation_server/layers/attention/common.py
index bd0717ce..b986a082 100644
--- a/server/text_generation_server/layers/attention/common.py
+++ b/server/text_generation_server/layers/attention/common.py
@@ -1,10 +1,10 @@
 from dataclasses import dataclass
-from text_generation_server.models.globals import FLASH_DECODING
+from text_generation_server.models.globals import FLASH_DECODING, FLASH_INFER
 import torch
 from typing import Optional
 
 
-if FLASH_DECODING:
+if FLASH_DECODING or FLASH_INFER:
 
     @dataclass
     class Seqlen:
diff --git a/server/text_generation_server/layers/attention/cuda.py b/server/text_generation_server/layers/attention/cuda.py
index 96b654d0..1b8e9209 100644
--- a/server/text_generation_server/layers/attention/cuda.py
+++ b/server/text_generation_server/layers/attention/cuda.py
@@ -1,6 +1,10 @@
 import torch
 from text_generation_server.utils.import_utils import SYSTEM
-from text_generation_server.models.globals import FLASH_DECODING, BLOCK_SIZE
+from text_generation_server.models.globals import (
+    FLASH_DECODING,
+    BLOCK_SIZE,
+    FLASH_INFER,
+)
 from text_generation_server.layers.attention import Seqlen
 from typing import Optional
 
@@ -23,7 +27,7 @@ def reshape_and_cache(
     value_cache: torch.Tensor,
     slots: torch.Tensor,
 ):
-    if FLASH_DECODING:
+    if FLASH_DECODING or FLASH_INFER:
         shape = key_cache.shape
         key_cache.view(-1, shape[-2], shape[-1])[slots] = key
         value_cache.view(-1, shape[-2], shape[-1])[slots] = value
@@ -72,7 +76,16 @@ def paged_attention(
     # V1 to avoid the overhead of reduction. Also, if the number of
     # sequences or heads is large, we use V1 since there is enough work
     # to parallelize.
-    if FLASH_DECODING:
+    if FLASH_INFER:
+        from text_generation_server.layers.attention.flash_infer import decode_state
+
+        return decode_state.get().forward(
+            query.contiguous(),
+            paged_kv_cache=(key_cache, value_cache),
+            logits_soft_cap=softcap,
+            sm_scale=softmax_scale,
+        )
+    elif FLASH_DECODING:
         max_q = 1
         max_k = max_s
         import flash_attn_2_cuda
@@ -206,7 +219,32 @@ except ImportError:
 
 SUPPORTS_WINDOWING = V2
 
-if V2:
+if FLASH_INFER:
+
+    def attention(
+        q,
+        k,
+        v,
+        cu_seqlens,
+        max_s,
+        softmax_scale,
+        window_size_left=-1,
+        causal=True,
+        softcap=0.0,
+    ):
+        from text_generation_server.layers.attention.flash_infer import prefill_state
+
+        return prefill_state.get().forward(
+            q,
+            k,
+            v,
+            causal=causal,
+            window_left=window_size_left,
+            logits_soft_cap=softcap,
+            sm_scale=softmax_scale,
+        )
+
+elif V2:
 
     def attention(
         q,
diff --git a/server/text_generation_server/layers/attention/flash_infer.py b/server/text_generation_server/layers/attention/flash_infer.py
new file mode 100644
index 00000000..56b53b2c
--- /dev/null
+++ b/server/text_generation_server/layers/attention/flash_infer.py
@@ -0,0 +1,164 @@
+from typing import Optional
+from contextvars import ContextVar
+from contextlib import contextmanager
+
+import flashinfer
+import torch
+
+prefill_state: ContextVar[flashinfer.BatchPrefillWithRaggedKVCacheWrapper] = ContextVar(
+    "prefill_state"
+)
+
+decode_state: ContextVar[flashinfer.BatchDecodeWithPagedKVCacheWrapper] = ContextVar(
+    "decode_state"
+)
+
+workspace: Optional[torch.Tensor] = None
+
+
+def get_workspace(device):
+    """Get shared flashinfer workspace."""
+    global workspace
+    if workspace is None:
+        workspace = torch.empty(128 * 1024 * 1024, dtype=torch.uint8, device=device)
+    return workspace
+
+
+def create_prefill_state(
+    *,
+    device: torch.device,
+):
+    """Create a prefill state."""
+    workspace_buffer = get_workspace(device)
+    return flashinfer.BatchPrefillWithRaggedKVCacheWrapper(
+        workspace_buffer, kv_layout="NHD", use_cuda_graph=False
+    )
+
+
+@contextmanager
+def use_prefill_state(
+    *,
+    state: flashinfer.BatchPrefillWithRaggedKVCacheWrapper,
+    cu_seqlens: torch.Tensor,
+    num_heads: int,
+    num_kv_heads: int,
+    head_size: int,
+    query_dtype: str = "float16",
+):
+    """
+    Context manager to set the active flashinfer prefill state to the given
+    `state` and parameters. This state will be used by all calls to the
+    `attention` function while the context manager is active.
+    """
+
+    token = prefill_state.set(state)
+    try:
+        state.begin_forward(
+            qo_indptr=cu_seqlens,
+            kv_indptr=cu_seqlens,
+            num_qo_heads=num_heads,
+            num_kv_heads=num_kv_heads,
+            head_dim=head_size,
+            q_data_type=query_dtype,
+        )
+        yield
+    finally:
+        state.end_forward()
+        if token is not None:
+            prefill_state.reset(token)
+
+
+def create_decode_state(
+    *,
+    device: torch.device,
+    num_heads: int,
+    num_kv_heads: int,
+):
+    """Create a decode state."""
+    workspace_buffer = get_workspace(device)
+    return flashinfer.BatchDecodeWithPagedKVCacheWrapper(
+        workspace_buffer,
+        kv_layout="NHD",
+        use_cuda_graph=False,
+        use_tensor_cores=num_heads // num_kv_heads > 4,
+    )
+
+
+def create_decode_state_cuda_graphs(
+    *,
+    device: torch.device,
+    block_tables: torch.Tensor,
+    block_tables_ptr: torch.Tensor,
+    last_page_len: torch.Tensor,
+    num_heads: int,
+    num_kv_heads: int,
+):
+    """
+    Create a decode state for use with CUDA Graphs. `block_tables`,
+    `block_tables_ptr`, and `last_page_len` are used in CUDA Graphs and are
+    therefore stored as part of the state.
+    """
+    workspace_buffer = get_workspace(device)
+    return flashinfer.BatchDecodeWithPagedKVCacheWrapper(
+        workspace_buffer,
+        kv_layout="NHD",
+        use_cuda_graph=True,
+        paged_kv_indices_buffer=block_tables,
+        paged_kv_indptr_buffer=block_tables_ptr,
+        paged_kv_last_page_len_buffer=last_page_len,
+        use_tensor_cores=num_heads // num_kv_heads > 4,
+    )
+
+
+@contextmanager
+def use_decode_state(
+    *,
+    state: flashinfer.BatchDecodeWithPagedKVCacheWrapper,
+    input_lengths: torch.Tensor,
+    block_tables: torch.Tensor,
+    num_heads: int,
+    num_kv_heads: int,
+    head_size: int,
+    page_size: int,
+    query_dtype: str = "float16",
+):
+    """
+    Context manager to set the active flashinfer decoding state to the given
+    `state` and parameters. This state will be used by all calls to the
+    `paged_attention` function while the context manager is active.
+    """
+    indptr = torch.zeros(
+        input_lengths.shape[0] + 1, device=input_lengths.device, dtype=torch.int32
+    )
+    # Round up to page size and then calculate the cumulative sum to get
+    # the indices into the block table.
+    torch.add(input_lengths, page_size - 1, out=indptr[1:])
+    indptr[1:].div_(page_size, rounding_mode="floor")
+    indptr[1:].cumsum_(-1)
+
+    # Get the lengths of the last page in a block.
+    last_page_len = torch.empty(
+        input_lengths.shape[0], dtype=torch.int32, device=input_lengths.device
+    )
+    torch.sub(input_lengths, 1, out=last_page_len)
+    last_page_len.remainder_(page_size)
+    last_page_len += 1
+
+    token = decode_state.set(state)
+
+    try:
+        state.begin_forward(
+            indptr=indptr,
+            indices=block_tables,
+            last_page_len=last_page_len,
+            num_qo_heads=num_heads,
+            num_kv_heads=num_kv_heads,
+            head_dim=head_size,
+            page_size=page_size,
+            q_data_type=query_dtype,
+        )
+        yield
+    finally:
+        state.end_forward()
+        if token is not None:
+            decode_state.reset(token)
diff --git a/server/text_generation_server/models/flash_causal_lm.py b/server/text_generation_server/models/flash_causal_lm.py
index 36bb2662..12aa7dcd 100644
--- a/server/text_generation_server/models/flash_causal_lm.py
+++ b/server/text_generation_server/models/flash_causal_lm.py
@@ -1,3 +1,4 @@
+from contextlib import nullcontext
 import math
 import os
 import time
@@ -15,7 +16,7 @@ from transformers import (
     AutoTokenizer,
     GenerationConfig,
 )
-from typing import Iterable, Optional, Tuple, List, Type, Dict
+from typing import Any, ContextManager, Iterable, Optional, Tuple, List, Type, Dict
 
 from text_generation_server.adapters import AdapterBatchData, AdapterBatchMetadata
 from huggingface_hub.constants import HUGGINGFACE_HUB_CACHE
@@ -40,6 +41,7 @@ from text_generation_server.pb import generate_pb2
 from text_generation_server.models.globals import (
     MEM_POOL,
     FLASH_DECODING,
+    FLASH_INFER,
     BLOCK_SIZE,
     CUDA_GRAPHS,
     get_adapter_to_index,
@@ -907,6 +909,7 @@ class FlashCausalLM(Model):
             config.sliding_window = None
 
         self.num_layers = config.num_hidden_layers
+        self.num_heads = config.num_attention_heads
         # Validation is done in the model itself
         if num_kv_heads is None:
             num_kv_heads = getattr(config, "num_key_value_heads", None)
@@ -935,6 +938,21 @@ class FlashCausalLM(Model):
         self.cuda_graphs = {}
         self.kv_cache = []
 
+        if FLASH_INFER:
+            from text_generation_server.layers.attention.flash_infer import (
+                create_prefill_state,
+                create_decode_state,
+            )
+
+            self.prefill_state = create_prefill_state(device=device)
+
+            if not CUDA_GRAPHS:
+                self.decode_state = create_decode_state(
+                    device=device,
+                    num_heads=self.num_heads,
+                    num_kv_heads=self.num_kv_heads,
+                )
+
         super().__init__(
             model_id=model_id,
             model=model,
@@ -972,7 +990,7 @@ class FlashCausalLM(Model):
         else:
             x = BLOCK_SIZE // element_size
 
-        if FLASH_DECODING:
+        if FLASH_DECODING or FLASH_INFER:
             self.kv_cache = [
                 (
                     torch.empty(
@@ -1044,38 +1062,66 @@ class FlashCausalLM(Model):
         graph = torch.cuda.CUDAGraph()
         self.cuda_graphs[bs]["graph"] = graph
 
+        if FLASH_INFER:
+            from text_generation_server.layers.attention.flash_infer import (
+                create_decode_state_cuda_graphs,
+            )
+
+            block_tables_ptr = torch.zeros(
+                bs + 1, dtype=torch.int32, device=self.device
+            )
+            last_page_len = torch.ones(bs, dtype=torch.int32, device=self.device)
+            state = create_decode_state_cuda_graphs(
+                device=input_ids.device,
+                block_tables=block_tables.view(-1),
+                block_tables_ptr=block_tables_ptr,
+                last_page_len=last_page_len,
+                num_heads=self.num_heads,
+                num_kv_heads=self.num_kv_heads,
+            )
+            self.cuda_graphs[bs]["state"] = state
+        else:
+            state = None
+
         torch.cuda.synchronize()
         # Run once outside to warmup
-        self.model.forward(
-            input_ids=input_ids,
-            position_ids=position_ids,
-            cu_seqlen_prefill=None,
-            kv_cache=self.kv_cache,
+        with self._forward_context(
             block_tables=block_tables,
-            slots=slots,
-            input_lengths=input_lengths_,
-            max_s=max_s,
-            prefill_cache_indices=None,
-            lm_head_indices=None,
-        )
-        torch.cuda.synchronize()
-
-        with torch.cuda.graph(graph, pool=MEM_POOL):
-            input_lengths = Seqlen(input_lengths=input_lengths)
-            logits, speculative_logits = self.model.forward(
+            cu_seqlen_prefill=None,
+            input_lengths=input_lengths,
+            state=state,
+        ):
+            self.model.forward(
                 input_ids=input_ids,
                 position_ids=position_ids,
                 cu_seqlen_prefill=None,
                 kv_cache=self.kv_cache,
                 block_tables=block_tables,
                 slots=slots,
-                input_lengths=input_lengths,
+                input_lengths=input_lengths_,
                 max_s=max_s,
                 prefill_cache_indices=None,
                 lm_head_indices=None,
             )
-            self.cuda_graphs[bs]["logits"] = logits
-            self.cuda_graphs[bs]["speculative_logits"] = speculative_logits
+
+            torch.cuda.synchronize()
+
+            with torch.cuda.graph(graph, pool=MEM_POOL):
+                input_lengths = Seqlen(input_lengths=input_lengths)
+                logits, speculative_logits = self.model.forward(
+                    input_ids=input_ids,
+                    position_ids=position_ids,
+                    cu_seqlen_prefill=None,
+                    kv_cache=self.kv_cache,
+                    block_tables=block_tables,
+                    slots=slots,
+                    input_lengths=input_lengths,
+                    max_s=max_s,
+                    prefill_cache_indices=None,
+                    lm_head_indices=None,
+                )
+                self.cuda_graphs[bs]["logits"] = logits
+                self.cuda_graphs[bs]["speculative_logits"] = speculative_logits
         torch.cuda.synchronize()
 
     def warmup(self, batch: FlashCausalLMBatch):
@@ -1295,23 +1341,28 @@ class FlashCausalLM(Model):
             cuda_graph = None
 
         if cu_seqlen_prefill is not None or cuda_graph is None:
-            input_lengths = Seqlen(input_lengths=input_lengths)
-            logits, speculative_logits = self.model.forward(
-                input_ids=input_ids,
-                position_ids=position_ids,
-                cu_seqlen_prefill=cu_seqlen_prefill,
-                kv_cache=kv_cache,
+            with self._forward_context(
                 block_tables=block_tables,
-                slots=slots,
+                cu_seqlen_prefill=cu_seqlen_prefill,
                 input_lengths=input_lengths,
-                max_s=max_s,
-                prefill_cache_indices=batch.prefill_cache_indices,
-                lm_head_indices=lm_head_indices,
-                adapter_data=adapter_data,
-            )
-            if batch.prefill_cache_indices is not None:
-                batch.prefill_cache_indices = None
-            return logits, speculative_logits
+            ):
+                input_lengths = Seqlen(input_lengths=input_lengths)
+                logits, speculative_logits = self.model.forward(
+                    input_ids=input_ids,
+                    position_ids=position_ids,
+                    cu_seqlen_prefill=cu_seqlen_prefill,
+                    kv_cache=kv_cache,
+                    block_tables=block_tables,
+                    slots=slots,
+                    input_lengths=input_lengths,
+                    max_s=max_s,
+                    prefill_cache_indices=batch.prefill_cache_indices,
+                    lm_head_indices=lm_head_indices,
+                    adapter_data=adapter_data,
+                )
+                if batch.prefill_cache_indices is not None:
+                    batch.prefill_cache_indices = None
+                return logits, speculative_logits
 
         # Copy inputs to the static inputs of the cuda graph
         # Static inputs are potentially padded
@@ -1325,8 +1376,16 @@ class FlashCausalLM(Model):
         cuda_graph["input_lengths"].zero_()
         cuda_graph["input_lengths"][: input_lengths.shape[0]] = input_lengths
 
-        # Replay the graph
-        cuda_graph["graph"].replay()
+        state = cuda_graph.get("state")
+        with self._forward_context(
+            block_tables=block_tables,
+            cu_seqlen_prefill=None,
+            input_lengths=input_lengths,
+            state=state,
+        ):
+            # Replay the graph
+            cuda_graph["graph"].replay()
+
         # Slice output to the correct shape
         speculative_logits = (
             cuda_graph["speculative_logits"][:bs]
@@ -1698,3 +1757,39 @@ class FlashCausalLM(Model):
         forward_ns = start_decode - start
         decode_ns = time.time_ns() - start_decode
         return generations, batch, (forward_ns, decode_ns)
+
+    def _forward_context(
+        self,
+        *,
+        block_tables: torch.Tensor,
+        cu_seqlen_prefill: Optional[torch.Tensor],
+        input_lengths: torch.Tensor,
+        state: Optional[Any] = None,
+    ) -> ContextManager:
+        if not FLASH_INFER:
+            return nullcontext()
+
+        from text_generation_server.layers.attention.flash_infer import (
+            use_decode_state,
+            use_prefill_state,
+        )
+
+        if cu_seqlen_prefill is not None:
+            return use_prefill_state(
+                state=state if state is not None else self.prefill_state,
+                cu_seqlens=cu_seqlen_prefill,
+                num_heads=self.num_heads,
+                num_kv_heads=self.num_kv_heads,
+                head_size=self.head_size,
+            )
+        else:
+            assert input_lengths is not None
+            return use_decode_state(
+                state=state if state is not None else self.decode_state,
+                input_lengths=input_lengths,
+                block_tables=block_tables.view(-1),
+                num_heads=self.num_heads,
+                num_kv_heads=self.num_kv_heads,
+                head_size=self.head_size,
+                page_size=BLOCK_SIZE,
+            )
diff --git a/server/text_generation_server/models/globals.py b/server/text_generation_server/models/globals.py
index 8d2431db..42b43c87 100644
--- a/server/text_generation_server/models/globals.py
+++ b/server/text_generation_server/models/globals.py
@@ -5,6 +5,10 @@ from typing import Dict, Optional
 
 from text_generation_server.utils.log import log_master
 
+FLASH_INFER = os.getenv("FLASH_INFER") in {"1", "true", "True"}
+if FLASH_INFER:
+    log_master(logger.info, "Using FLASH_INFER")
+
 MEM_POOL = torch.cuda.graph_pool_handle() if torch.cuda.is_available() else None
 # This is overridden by the cli
 FLASH_DECODING = os.getenv("FLASH_DECODING") in {"1", "true", "True"}
@@ -12,6 +16,7 @@ BLOCK_SIZE: int = 256 if FLASH_DECODING else 16
 if FLASH_DECODING:
     log_master(logger.info, "Using FLASH_DECODING")
 
+
 cuda_graphs = os.getenv("CUDA_GRAPHS")
 if cuda_graphs is not None:
     try:

From c6d5039cd7c05c9d1323b94844ca30ab593f21f8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Dani=C3=ABl=20de=20Kok?= <me@danieldk.eu>
Date: Fri, 9 Aug 2024 12:32:37 +0200
Subject: [PATCH 19/72] Add experimental flake (#2384)

Add flake.nix
---
 flake.lock | 99 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 flake.nix  | 73 ++++++++++++++++++++++++++++++++++++++++
 2 files changed, 172 insertions(+)
 create mode 100644 flake.lock
 create mode 100644 flake.nix

diff --git a/flake.lock b/flake.lock
new file mode 100644
index 00000000..bf06240f
--- /dev/null
+++ b/flake.lock
@@ -0,0 +1,99 @@
+{
+  "nodes": {
+    "flake-compat": {
+      "locked": {
+        "lastModified": 1696426674,
+        "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=",
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
+    "flake-utils": {
+      "inputs": {
+        "systems": "systems"
+      },
+      "locked": {
+        "lastModified": 1710146030,
+        "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1723099294,
+        "narHash": "sha256-kkijy6sXo/SOhFw7ZEfYHbj1FJHxoeetOVOn3qNHc5o=",
+        "owner": "danieldk",
+        "repo": "nixpkgs",
+        "rev": "45892b6ec142eaf300d777926983a433b5842c88",
+        "type": "github"
+      },
+      "original": {
+        "owner": "danieldk",
+        "ref": "cudnn-9.3",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "flake-utils": "flake-utils",
+        "nixpkgs": [
+          "tgi-nix",
+          "nixpkgs"
+        ],
+        "tgi-nix": "tgi-nix"
+      }
+    },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
+    "tgi-nix": {
+      "inputs": {
+        "flake-compat": "flake-compat",
+        "nixpkgs": "nixpkgs"
+      },
+      "locked": {
+        "lastModified": 1723188417,
+        "narHash": "sha256-GXdFuRMU9N9W0CryUTJIWhJkGwQFHSR2EW5xR0ZyBjk=",
+        "owner": "danieldk",
+        "repo": "tgi-nix",
+        "rev": "491db7e234ecf79513ddb94da6ecc14167b9c0b3",
+        "type": "github"
+      },
+      "original": {
+        "owner": "danieldk",
+        "repo": "tgi-nix",
+        "type": "github"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}
diff --git a/flake.nix b/flake.nix
new file mode 100644
index 00000000..c17950c1
--- /dev/null
+++ b/flake.nix
@@ -0,0 +1,73 @@
+{
+  inputs = {
+    tgi-nix.url = "github:danieldk/tgi-nix";
+    nixpkgs.follows = "tgi-nix/nixpkgs";
+    flake-utils.url = "github:numtide/flake-utils";
+  };
+  outputs =
+    {
+      self,
+      nixpkgs,
+      flake-utils,
+      tgi-nix,
+    }:
+    flake-utils.lib.eachDefaultSystem (
+      system:
+      let
+        config = {
+          allowUnfree = true;
+          cudaSupport = true;
+        };
+        pkgs = import nixpkgs {
+          inherit config system;
+          overlays = [ tgi-nix.overlay ];
+        };
+      in
+      {
+        devShells.default =
+          with pkgs;
+          mkShell {
+            buildInputs =
+              [
+                cargo
+                openssl.dev
+                pkg-config
+              ]
+              ++ (with python3.pkgs; [
+                venvShellHook
+                pip
+
+                einops
+                fbgemm-gpu
+                flash-attn
+                flash-attn-layer-norm
+                flash-attn-rotary
+                grpc-interceptor
+                grpcio-reflection
+                grpcio-status
+                hf-transfer
+                loguru
+                marlin-kernels
+                opentelemetry-api
+                opentelemetry-exporter-otlp
+                opentelemetry-instrumentation-grpc
+                opentelemetry-semantic-conventions
+                peft
+                tokenizers
+                torch
+                transformers
+                vllm
+              ]);
+
+            venvDir = "./.venv";
+
+            postVenv = ''
+              unset SOURCE_DATE_EPOCH
+            '';
+            postShellHook = ''
+              unset SOURCE_DATE_EPOCH
+            '';
+          };
+      }
+    );
+}

From 952b450a3b4ddf7e41c24f807357a158fd3eae0c Mon Sep 17 00:00:00 2001
From: Nicolas Patry <patry.nicolas@protonmail.com>
Date: Fri, 9 Aug 2024 14:25:44 +0200
Subject: [PATCH 20/72] Using HF_HOME instead of CACHE to get token read in
 addition to models. (#2288)

---
 Dockerfile       | 2 +-
 Dockerfile_amd   | 2 +-
 Dockerfile_intel | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 0d57e38d..c68f76f6 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -191,7 +191,7 @@ ENV PATH=/opt/conda/bin:$PATH \
     CONDA_PREFIX=/opt/conda
 
 # Text Generation Inference base env
-ENV HUGGINGFACE_HUB_CACHE=/data \
+ENV HF_HOME=/data \
     HF_HUB_ENABLE_HF_TRANSFER=1 \
     PORT=80
 
diff --git a/Dockerfile_amd b/Dockerfile_amd
index 51231638..cdad0d28 100644
--- a/Dockerfile_amd
+++ b/Dockerfile_amd
@@ -167,7 +167,7 @@ RUN python setup.py build
 FROM base AS base-copy
 
 # Text Generation Inference base env
-ENV HUGGINGFACE_HUB_CACHE=/data \
+ENV HF_HOME=/data \
     HF_HUB_ENABLE_HF_TRANSFER=1 \
     PORT=80
 
diff --git a/Dockerfile_intel b/Dockerfile_intel
index d20f0a01..158c5a89 100644
--- a/Dockerfile_intel
+++ b/Dockerfile_intel
@@ -57,7 +57,7 @@ RUN wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRO
 RUN apt-get update && apt install -y intel-basekit xpu-smi cmake python3-dev ninja-build pciutils
 
 # Text Generation Inference base env
-ENV HUGGINGFACE_HUB_CACHE=/data \
+ENV HF_HOME=/data \
     HF_HUB_ENABLE_HF_TRANSFER=1 \
     PORT=80
 

From 977534bcb8b078ed7ee9df1ae8038d083cd1a58d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Dani=C3=ABl=20de=20Kok?= <me@danieldk.eu>
Date: Fri, 9 Aug 2024 14:56:20 +0200
Subject: [PATCH 21/72] flake: add fmt and clippy (#2389)

---
 flake.nix | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/flake.nix b/flake.nix
index c17950c1..a67879fc 100644
--- a/flake.nix
+++ b/flake.nix
@@ -30,8 +30,10 @@
             buildInputs =
               [
                 cargo
+                clippy
                 openssl.dev
                 pkg-config
+                rustfmt
               ]
               ++ (with python3.pkgs; [
                 venvShellHook

From b2b9c427246d895a943fa6a5f8e9b702eff03559 Mon Sep 17 00:00:00 2001
From: Vaibhav Srivastav <vaibhavs10@gmail.com>
Date: Fri, 9 Aug 2024 15:01:34 +0200
Subject: [PATCH 22/72] Update documentation for Supported models (#2386)

* Minor doc fixes

* up.

* Other minor updates.
---
 README.md                                     | 34 ++++++++++++++++---
 docs/source/conceptual/quantization.md        |  4 +--
 docs/source/quicktour.md                      |  2 +-
 docs/source/supported_models.md               |  8 ++---
 .../text_generation_server/models/__init__.py |  6 ++--
 update_doc.py                                 |  2 +-
 6 files changed, 41 insertions(+), 15 deletions(-)

diff --git a/README.md b/README.md
index cf7f1d22..803e9172 100644
--- a/README.md
+++ b/README.md
@@ -13,7 +13,7 @@
   <img alt="Swagger API documentation" src="https://img.shields.io/badge/API-Swagger-informational">
 </a>
 
-A Rust, Python and gRPC server for text generation inference. Used in production at [HuggingFace](https://huggingface.co)
+A Rust, Python and gRPC server for text generation inference. Used in production at [Hugging Face](https://huggingface.co)
 to power Hugging Chat, the Inference API and Inference Endpoint.
 
 </div>
@@ -42,6 +42,7 @@ Text Generation Inference (TGI) is a toolkit for deploying and serving Large Lan
 - Tensor Parallelism for faster inference on multiple GPUs
 - Token streaming using Server-Sent Events (SSE)
 - Continuous batching of incoming requests for increased total throughput
+- [Messages API](https://huggingface.co/docs/text-generation-inference/en/messages_api) compatible with Open AI Chat Completion API
 - Optimized transformers code for inference using [Flash Attention](https://github.com/HazyResearch/flash-attention) and [Paged Attention](https://github.com/vllm-project/vllm) on the most popular architectures
 - Quantization with :
   - [bitsandbytes](https://github.com/TimDettmers/bitsandbytes)
@@ -49,7 +50,7 @@ Text Generation Inference (TGI) is a toolkit for deploying and serving Large Lan
   - [EETQ](https://github.com/NetEase-FuXi/EETQ)
   - [AWQ](https://github.com/casper-hansen/AutoAWQ)
   - [Marlin](https://github.com/IST-DASLab/marlin)
-  - [fp8]()
+  - [fp8](https://developer.nvidia.com/blog/nvidia-arm-and-intel-publish-fp8-specification-for-standardization-as-an-interchange-format-for-ai/)
 - [Safetensors](https://github.com/huggingface/safetensors) weight loading
 - Watermarking with [A Watermark for Large Language Models](https://arxiv.org/abs/2301.10226)
 - Logits warper (temperature scaling, top-p, top-k, repetition penalty, more details see [transformers.LogitsProcessor](https://huggingface.co/docs/transformers/internal/generation_utils#transformers.LogitsProcessor))
@@ -94,6 +95,29 @@ curl 127.0.0.1:8080/generate_stream \
     -H 'Content-Type: application/json'
 ```
 
+You can also use [TGI's Messages API](https://huggingface.co/docs/text-generation-inference/en/messages_api) to obtain Open AI Chat Completion API compatible responses.
+
+```bash
+curl localhost:3000/v1/chat/completions \
+    -X POST \
+    -d '{
+  "model": "tgi",
+  "messages": [
+    {
+      "role": "system",
+      "content": "You are a helpful assistant."
+    },
+    {
+      "role": "user",
+      "content": "What is deep learning?"
+    }
+  ],
+  "stream": true,
+  "max_tokens": 20
+}' \
+    -H 'Content-Type: application/json'
+```
+
 **Note:** To use NVIDIA GPUs, you need to install the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html). We also recommend using NVIDIA drivers with CUDA version 12.2 or higher. For running the Docker container on a machine with no GPUs or CUDA support, it is enough to remove the `--gpus all` flag and add `--disable-custom-kernels`, please note CPU is not the intended platform for this project, so performance might be subpar.
 
 **Note:** TGI supports AMD Instinct MI210 and MI250 GPUs. Details can be found in the [Supported Hardware documentation](https://huggingface.co/docs/text-generation-inference/supported_models#supported-hardware). To use AMD GPUs, please use `docker run --device /dev/kfd --device /dev/dri --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingface/text-generation-inference:2.2.0-rocm --model-id $model` instead of the command above.
@@ -122,7 +146,7 @@ For example, if you want to serve the gated Llama V2 model variants:
 or with Docker:
 
 ```shell
-model=meta-llama/Llama-2-7b-chat-hf
+model=meta-llama/Meta-Llama-3.1-8B-Instruct
 volume=$PWD/data # share a volume with the Docker container to avoid downloading weights every run
 token=<your cli READ token>
 
@@ -234,7 +258,7 @@ text-generation-launcher --model-id mistralai/Mistral-7B-Instruct-v0.2
 
 ### Quantization
 
-You can also quantize the weights with bitsandbytes to reduce the VRAM requirement:
+You can also run pre-quantized weights (AWQ, GPTQ, Marlin) or on-the-fly quantize weights with bitsandbytes, EETQ, fp8, to reduce the VRAM requirement:
 
 ```shell
 text-generation-launcher --model-id mistralai/Mistral-7B-Instruct-v0.2 --quantize
@@ -242,6 +266,8 @@ text-generation-launcher --model-id mistralai/Mistral-7B-Instruct-v0.2 --quantiz
 
 4bit quantization is available using the [NF4 and FP4 data types from bitsandbytes](https://arxiv.org/pdf/2305.14314.pdf). It can be enabled by providing `--quantize bitsandbytes-nf4` or `--quantize bitsandbytes-fp4` as a command line argument to `text-generation-launcher`.
 
+Read more about quantization in the [Quantization documentation](https://huggingface.co/docs/text-generation-inference/en/conceptual/quantization).
+
 ## Develop
 
 ```shell
diff --git a/docs/source/conceptual/quantization.md b/docs/source/conceptual/quantization.md
index 7507687f..a1ebe7e7 100644
--- a/docs/source/conceptual/quantization.md
+++ b/docs/source/conceptual/quantization.md
@@ -11,7 +11,7 @@ We recommend using the official quantization scripts for creating your quants:
 
 For on-the-fly quantization you simply need to pass one of the supported quantization types and TGI takes care of the rest. 
 
-## Quantization with bitsandbytes
+## Quantization with bitsandbytes, EETQ & fp8
 
 bitsandbytes is a library used to apply 8-bit and 4-bit quantization to models. Unlike GPTQ quantization, bitsandbytes doesn't require a calibration dataset or any post-processing – weights are automatically quantized on load. However, inference with bitsandbytes is slower than GPTQ or FP16 precision.
 
@@ -32,7 +32,7 @@ docker run --gpus all --shm-size 1g -p 8080:80 -v $volume:/data ghcr.io/huggingf
 
 You can get more information about 8-bit quantization by reading this [blog post](https://huggingface.co/blog/hf-bitsandbytes-integration), and 4-bit quantization by reading [this blog post](https://huggingface.co/blog/4bit-transformers-bitsandbytes).
 
-Use `eetq` or `fp8` for other quantization schemes.
+Similarly you can use pass you can pass `--quantize eetq` or `--quantize fp8` for respective quantization schemes.
 
 In addition to this, TGI allows creating GPTQ quants directly by passing the model weights and a calibration dataset.
 
diff --git a/docs/source/quicktour.md b/docs/source/quicktour.md
index 2313c69b..18e1a107 100644
--- a/docs/source/quicktour.md
+++ b/docs/source/quicktour.md
@@ -21,7 +21,7 @@ TGI supports various hardware. Make sure to check the [Using TGI with Nvidia GPU
 
 ## Consuming TGI
 
-Once TGI is running, you can use the `generate` endpoint by doing requests. To learn more about how to query the endpoints, check the [Consuming TGI](./basic_tutorials/consuming_tgi) section, where we show examples with utility libraries and UIs. Below you can see a simple snippet to query the endpoint.
+Once TGI is running, you can use the `generate` endpoint or the Open AI Chat Completion API compatible [Messages API](https://huggingface.co/docs/text-generation-inference/en/messages_api) by doing requests. To learn more about how to query the endpoints, check the [Consuming TGI](./basic_tutorials/consuming_tgi) section, where we show examples with utility libraries and UIs. Below you can see a simple snippet to query the endpoint.
 
 <inferencesnippet>
 <python>
diff --git a/docs/source/supported_models.md b/docs/source/supported_models.md
index b78104df..832f88ef 100644
--- a/docs/source/supported_models.md
+++ b/docs/source/supported_models.md
@@ -1,22 +1,22 @@
 
 # Supported Models and Hardware
 
-Text Generation Inference enables serving optimized models on specific hardware for the highest performance. The following sections list which models are hardware are supported.
+Text Generation Inference enables serving optimized models on specific hardware for the highest performance. The following sections list which models (VLMs & LLMs) are supported.
 
 ## Supported Models
 
 - [Deepseek V2](https://huggingface.co/deepseek-ai/DeepSeek-V2)
 - [Idefics 2](https://huggingface.co/HuggingFaceM4/idefics2-8b) (Multimodal)
 - [Llava Next (1.6)](https://huggingface.co/llava-hf/llava-v1.6-vicuna-13b-hf) (Multimodal)
-- [Llama](https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct)
+- [Llama](https://huggingface.co/collections/meta-llama/llama-31-669fc079a0c406a149a5738f)
 - [Phi 3](https://huggingface.co/microsoft/Phi-3-mini-4k-instruct)
 - [Gemma](https://huggingface.co/google/gemma-7b)
 - [PaliGemma](https://huggingface.co/google/paligemma-3b-pt-224)
-- [Gemma2](https://huggingface.co/google/gemma2-9b)
+- [Gemma2](https://huggingface.co/collections/google/gemma-2-release-667d6600fd5220e7b967f315)
 - [Cohere](https://huggingface.co/CohereForAI/c4ai-command-r-plus)
 - [Dbrx](https://huggingface.co/databricks/dbrx-instruct)
 - [Mamba](https://huggingface.co/state-spaces/mamba-2.8b-slimpj)
-- [Mistral](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2)
+- [Mistral](https://huggingface.co/mistralai/Mistral-Nemo-Instruct-2407)
 - [Mixtral](https://huggingface.co/mistralai/Mixtral-8x22B-Instruct-v0.1)
 - [Gpt Bigcode](https://huggingface.co/bigcode/gpt_bigcode-santacoder)
 - [Phi](https://huggingface.co/microsoft/phi-1_5)
diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py
index da14d083..960b426b 100644
--- a/server/text_generation_server/models/__init__.py
+++ b/server/text_generation_server/models/__init__.py
@@ -180,7 +180,7 @@ class ModelType(enum.Enum):
     LLAMA = {
         "type": "llama",
         "name": "Llama",
-        "url": "https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct",
+        "url": "https://huggingface.co/collections/meta-llama/llama-31-669fc079a0c406a149a5738f",
     }
     PHI3 = {
         "type": "phi3",
@@ -200,7 +200,7 @@ class ModelType(enum.Enum):
     GEMMA2 = {
         "type": "gemma2",
         "name": "Gemma2",
-        "url": "https://huggingface.co/google/gemma2-9b",
+        "url": "https://huggingface.co/collections/google/gemma-2-release-667d6600fd5220e7b967f315",
     }
     COHERE = {
         "type": "cohere",
@@ -220,7 +220,7 @@ class ModelType(enum.Enum):
     MISTRAL = {
         "type": "mistral",
         "name": "Mistral",
-        "url": "https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2",
+        "url": "https://huggingface.co/mistralai/Mistral-Nemo-Instruct-2407",
     }
     MIXTRAL = {
         "type": "mixtral",
diff --git a/update_doc.py b/update_doc.py
index 428d4452..e887e1c6 100644
--- a/update_doc.py
+++ b/update_doc.py
@@ -7,7 +7,7 @@ import os
 TEMPLATE = """
 # Supported Models and Hardware
 
-Text Generation Inference enables serving optimized models on specific hardware for the highest performance. The following sections list which models are hardware are supported.
+Text Generation Inference enables serving optimized models on specific hardware for the highest performance. The following sections list which models (VLMs & LLMs) are supported.
 
 ## Supported Models
 

From 6e127dcc9616a0e39efd95a3e08324138e8c4df7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Dani=C3=ABl=20de=20Kok?= <me@danieldk.eu>
Date: Fri, 9 Aug 2024 15:24:21 +0200
Subject: [PATCH 23/72] flake: use rust-overlay (#2390)

---
 flake.lock | 22 ++++++++++++++++++++++
 flake.nix  | 19 +++++++++++++++----
 2 files changed, 37 insertions(+), 4 deletions(-)

diff --git a/flake.lock b/flake.lock
index bf06240f..043218e8 100644
--- a/flake.lock
+++ b/flake.lock
@@ -56,9 +56,31 @@
           "tgi-nix",
           "nixpkgs"
         ],
+        "rust-overlay": "rust-overlay",
         "tgi-nix": "tgi-nix"
       }
     },
+    "rust-overlay": {
+      "inputs": {
+        "nixpkgs": [
+          "tgi-nix",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1723170066,
+        "narHash": "sha256-SFkQfOA+8AIYJsPlQtxNP+z5jRLfz91z/aOrV94pPmw=",
+        "owner": "oxalica",
+        "repo": "rust-overlay",
+        "rev": "fecfe4d7c96fea2982c7907997b387a6b52c1093",
+        "type": "github"
+      },
+      "original": {
+        "owner": "oxalica",
+        "repo": "rust-overlay",
+        "type": "github"
+      }
+    },
     "systems": {
       "locked": {
         "lastModified": 1681028828,
diff --git a/flake.nix b/flake.nix
index a67879fc..fdd67d00 100644
--- a/flake.nix
+++ b/flake.nix
@@ -3,12 +3,17 @@
     tgi-nix.url = "github:danieldk/tgi-nix";
     nixpkgs.follows = "tgi-nix/nixpkgs";
     flake-utils.url = "github:numtide/flake-utils";
+    rust-overlay = {
+      url = "github:oxalica/rust-overlay";
+      inputs.nixpkgs.follows = "tgi-nix/nixpkgs";
+    };
   };
   outputs =
     {
       self,
       nixpkgs,
       flake-utils,
+      rust-overlay,
       tgi-nix,
     }:
     flake-utils.lib.eachDefaultSystem (
@@ -20,7 +25,10 @@
         };
         pkgs = import nixpkgs {
           inherit config system;
-          overlays = [ tgi-nix.overlay ];
+          overlays = [
+            rust-overlay.overlays.default
+            tgi-nix.overlay
+          ];
         };
       in
       {
@@ -29,11 +37,14 @@
           mkShell {
             buildInputs =
               [
-                cargo
-                clippy
                 openssl.dev
                 pkg-config
-                rustfmt
+                (rust-bin.stable.latest.default.override {
+                  extensions = [
+                    "rust-analyzer"
+                    "rust-src"
+                  ];
+                })
               ]
               ++ (with python3.pkgs; [
                 venvShellHook

From 7a48a84784b74f9ea12cf04a3c4572c027cec2e4 Mon Sep 17 00:00:00 2001
From: Nicolas Patry <patry.nicolas@protonmail.com>
Date: Fri, 9 Aug 2024 16:41:17 +0200
Subject: [PATCH 24/72] Using an enum for flash backens
 (paged/flashdecoding/flashinfer) (#2385)

* Using an enum for flash backens (paged/flashdecoding/flashinfer)

* Early exit on server too.

* Clippy.

* Fix clippy and fmt.
---
 .gitignore                                    |  1 +
 backends/v3/src/backend.rs                    | 16 ++++++----
 docs/openapi.json                             |  2 +-
 docs/source/conceptual/quantization.md        |  4 +--
 launcher/src/main.rs                          |  2 +-
 router/src/infer/v2/scheduler.rs              | 18 ++++++++----
 router/src/lib.rs                             | 29 +++++++++++++++++++
 .../layers/attention/common.py                |  4 +--
 .../layers/attention/cuda.py                  | 11 ++++---
 .../layers/attention/rocm.py                  |  4 +--
 .../models/flash_causal_lm.py                 | 11 ++++---
 .../text_generation_server/models/globals.py  | 14 ++++-----
 12 files changed, 78 insertions(+), 38 deletions(-)

diff --git a/.gitignore b/.gitignore
index 0de8b848..bd9d9125 100644
--- a/.gitignore
+++ b/.gitignore
@@ -18,3 +18,4 @@ server/exllama_kernels/exllama_kernels/exllama_ext_hip.cpp
 
 data/
 load_tests/*.json
+server/fbgemmm
diff --git a/backends/v3/src/backend.rs b/backends/v3/src/backend.rs
index 6b3e0526..68ddf00b 100644
--- a/backends/v3/src/backend.rs
+++ b/backends/v3/src/backend.rs
@@ -6,7 +6,7 @@ use nohash_hasher::IntMap;
 use std::sync::Arc;
 use text_generation_router::infer::{Backend, GeneratedText, InferError, InferStreamResponse};
 use text_generation_router::validation::ValidGenerateRequest;
-use text_generation_router::{FinishReason, PrefillToken, Token};
+use text_generation_router::{Attention, FinishReason, PrefillToken, Token};
 use tokio::sync::mpsc::error::SendError;
 use tokio::sync::{mpsc, Notify};
 use tokio::time::Instant;
@@ -35,12 +35,18 @@ impl BackendV3 {
         window_size: Option<u32>,
         speculate: u32,
     ) -> Self {
-        let flashdecoding = if let Ok(flashdecoding) = std::env::var("FLASH_DECODING") {
-            matches!(flashdecoding.to_lowercase().as_str(), "1" | "true")
+        let attention = if let Ok(attention) = std::env::var("ATTENTION") {
+            attention
+                .parse()
+                .unwrap_or_else(|_| panic!("Invalid attention was specified :`{attention}`"))
         } else {
-            false
+            Attention::Paged
+        };
+        let block_size = if attention == Attention::FlashDecoding {
+            256
+        } else {
+            16
         };
-        let block_size = if flashdecoding { 256 } else { 16 };
 
         let queue = Queue::new(
             requires_padding,
diff --git a/docs/openapi.json b/docs/openapi.json
index 9d281a48..ed9b0b96 100644
--- a/docs/openapi.json
+++ b/docs/openapi.json
@@ -2080,4 +2080,4 @@
       "description": "Hugging Face Text Generation Inference API"
     }
   ]
-}
\ No newline at end of file
+}
diff --git a/docs/source/conceptual/quantization.md b/docs/source/conceptual/quantization.md
index a1ebe7e7..b7672a9f 100644
--- a/docs/source/conceptual/quantization.md
+++ b/docs/source/conceptual/quantization.md
@@ -9,7 +9,7 @@ We recommend using the official quantization scripts for creating your quants:
 2. [GPTQ/ Marlin](https://github.com/AutoGPTQ/AutoGPTQ/blob/main/examples/quantization/basic_usage.py)
 3. [EXL2](https://github.com/turboderp/exllamav2/blob/master/doc/convert.md)
 
-For on-the-fly quantization you simply need to pass one of the supported quantization types and TGI takes care of the rest. 
+For on-the-fly quantization you simply need to pass one of the supported quantization types and TGI takes care of the rest.
 
 ## Quantization with bitsandbytes, EETQ & fp8
 
@@ -69,4 +69,4 @@ text-generation-launcher --model-id /data/falcon-40b-gptq/ --sharded true --num-
 You can learn more about the quantization options by running `text-generation-server quantize --help`.
 
 If you wish to do more with GPTQ models (e.g. train an adapter on top), you can read about transformers GPTQ integration [here](https://huggingface.co/blog/gptq-integration).
-You can learn more about GPTQ from the [paper](https://arxiv.org/pdf/2210.17323.pdf).
\ No newline at end of file
+You can learn more about GPTQ from the [paper](https://arxiv.org/pdf/2210.17323.pdf).
diff --git a/launcher/src/main.rs b/launcher/src/main.rs
index 8acfda0c..a64b1d71 100644
--- a/launcher/src/main.rs
+++ b/launcher/src/main.rs
@@ -1461,7 +1461,7 @@ fn main() -> Result<(), LauncherError> {
 
         if config.model_type == Some("gemma2".to_string()) {
             tracing::info!("Forcing flash decoding because of softcap usage");
-            std::env::set_var("FLASH_DECODING", "1");
+            std::env::set_var("ATTENTION", "flashdecoding");
         }
         let config: Config = config.into();
 
diff --git a/router/src/infer/v2/scheduler.rs b/router/src/infer/v2/scheduler.rs
index cc333674..0e5fc8a3 100644
--- a/router/src/infer/v2/scheduler.rs
+++ b/router/src/infer/v2/scheduler.rs
@@ -1,10 +1,10 @@
 /// Batching and inference logic
 use crate::infer::v2::queue::{Entry, Queue};
 use crate::infer::{
-    Backend, GenerateStreamResponse, GeneratedText, InferError, InferStreamResponse,
+    Attention, Backend, GenerateStreamResponse, GeneratedText, InferError, InferStreamResponse,
 };
 use crate::validation::ValidGenerateRequest;
-use crate::{FinishReason, PrefillToken, Token};
+use crate::{Attention, FinishReason, PrefillToken, Token};
 use nohash_hasher::IntMap;
 use std::sync::{
     atomic::{AtomicBool, Ordering},
@@ -40,12 +40,18 @@ impl BackendV2 {
         generation_health: Arc<AtomicBool>,
     ) -> Self {
         // Infer shared state
-        let flashdecoding = if let Ok(flashdecoding) = std::env::var("FLASH_DECODING") {
-            matches!(flashdecoding.to_lowercase().as_str(), "1" | "true")
+        let attention = if let Ok(attention) = std::env::var("ATTENTION") {
+            attention
+                .parse()
+                .expect(&format!("Invalid attention was specified :`{attention}`"))
         } else {
-            false
+            Attention::Paged
+        };
+        let block_size = if attention == Attention::FlashDecoding {
+            256
+        } else {
+            16
         };
-        let block_size = if flashdecoding { 256 } else { 16 };
         let queue = Queue::new(requires_padding, block_size, window_size, speculate);
         let batching_task_notifier = Arc::new(Notify::new());
 
diff --git a/router/src/lib.rs b/router/src/lib.rs
index a956b058..66738706 100644
--- a/router/src/lib.rs
+++ b/router/src/lib.rs
@@ -15,6 +15,35 @@ use tracing::warn;
 use utoipa::ToSchema;
 use validation::Validation;
 
+#[derive(PartialEq)]
+pub enum Attention {
+    Paged,
+    FlashDecoding,
+    FlashInfer,
+}
+
+#[derive(Debug)]
+pub struct ParseError;
+
+impl std::fmt::Display for ParseError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "Cannot parse attention value")
+    }
+}
+impl std::error::Error for ParseError {}
+
+impl std::str::FromStr for Attention {
+    type Err = ParseError;
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s {
+            "paged" => Ok(Attention::Paged),
+            "flashdecoding" => Ok(Attention::FlashDecoding),
+            "flashinfer" => Ok(Attention::FlashInfer),
+            _ => Err(ParseError),
+        }
+    }
+}
+
 #[derive(Clone, Deserialize, ToSchema)]
 pub(crate) struct VertexInstance {
     #[schema(example = "What is Deep Learning?")]
diff --git a/server/text_generation_server/layers/attention/common.py b/server/text_generation_server/layers/attention/common.py
index b986a082..f162230c 100644
--- a/server/text_generation_server/layers/attention/common.py
+++ b/server/text_generation_server/layers/attention/common.py
@@ -1,10 +1,10 @@
 from dataclasses import dataclass
-from text_generation_server.models.globals import FLASH_DECODING, FLASH_INFER
+from text_generation_server.models.globals import ATTENTION
 import torch
 from typing import Optional
 
 
-if FLASH_DECODING or FLASH_INFER:
+if ATTENTION in {"flashinfer", "flashdecoding"}:
 
     @dataclass
     class Seqlen:
diff --git a/server/text_generation_server/layers/attention/cuda.py b/server/text_generation_server/layers/attention/cuda.py
index 1b8e9209..d039e1e7 100644
--- a/server/text_generation_server/layers/attention/cuda.py
+++ b/server/text_generation_server/layers/attention/cuda.py
@@ -1,9 +1,8 @@
 import torch
 from text_generation_server.utils.import_utils import SYSTEM
 from text_generation_server.models.globals import (
-    FLASH_DECODING,
+    ATTENTION,
     BLOCK_SIZE,
-    FLASH_INFER,
 )
 from text_generation_server.layers.attention import Seqlen
 from typing import Optional
@@ -27,7 +26,7 @@ def reshape_and_cache(
     value_cache: torch.Tensor,
     slots: torch.Tensor,
 ):
-    if FLASH_DECODING or FLASH_INFER:
+    if ATTENTION in {"flashdecoding", "flashinfer"}:
         shape = key_cache.shape
         key_cache.view(-1, shape[-2], shape[-1])[slots] = key
         value_cache.view(-1, shape[-2], shape[-1])[slots] = value
@@ -76,7 +75,7 @@ def paged_attention(
     # V1 to avoid the overhead of reduction. Also, if the number of
     # sequences or heads is large, we use V1 since there is enough work
     # to parallelize.
-    if FLASH_INFER:
+    if ATTENTION == "flashinfer":
         from text_generation_server.layers.attention.flash_infer import decode_state
 
         return decode_state.get().forward(
@@ -85,7 +84,7 @@ def paged_attention(
             logits_soft_cap=softcap,
             sm_scale=softmax_scale,
         )
-    elif FLASH_DECODING:
+    elif ATTENTION == "flashdecoding":
         max_q = 1
         max_k = max_s
         import flash_attn_2_cuda
@@ -219,7 +218,7 @@ except ImportError:
 
 SUPPORTS_WINDOWING = V2
 
-if FLASH_INFER:
+if ATTENTION == "flashinfer":
 
     def attention(
         q,
diff --git a/server/text_generation_server/layers/attention/rocm.py b/server/text_generation_server/layers/attention/rocm.py
index 69e64162..16ce8d2b 100644
--- a/server/text_generation_server/layers/attention/rocm.py
+++ b/server/text_generation_server/layers/attention/rocm.py
@@ -1,7 +1,7 @@
 import os
 import torch
 from text_generation_server.utils.import_utils import SYSTEM
-from text_generation_server.models.globals import FLASH_DECODING
+from text_generation_server.models.globals import ATTENTION
 from text_generation_server.layers.attention import Seqlen
 from text_generation_server.utils.log import log_master
 from loguru import logger
@@ -28,7 +28,7 @@ def reshape_and_cache(
     value_cache: torch.Tensor,
     slots: torch.Tensor,
 ):
-    if FLASH_DECODING:
+    if ATTENTION == "flashdecoding":
         shape = key_cache.shape
         key_cache.view(-1, shape[-2], shape[-1])[slots] = key
         value_cache.view(-1, shape[-2], shape[-1])[slots] = value
diff --git a/server/text_generation_server/models/flash_causal_lm.py b/server/text_generation_server/models/flash_causal_lm.py
index 12aa7dcd..21b66a68 100644
--- a/server/text_generation_server/models/flash_causal_lm.py
+++ b/server/text_generation_server/models/flash_causal_lm.py
@@ -40,8 +40,7 @@ from text_generation_server.models.types import (
 from text_generation_server.pb import generate_pb2
 from text_generation_server.models.globals import (
     MEM_POOL,
-    FLASH_DECODING,
-    FLASH_INFER,
+    ATTENTION,
     BLOCK_SIZE,
     CUDA_GRAPHS,
     get_adapter_to_index,
@@ -938,7 +937,7 @@ class FlashCausalLM(Model):
         self.cuda_graphs = {}
         self.kv_cache = []
 
-        if FLASH_INFER:
+        if ATTENTION == "flashinfer":
             from text_generation_server.layers.attention.flash_infer import (
                 create_prefill_state,
                 create_decode_state,
@@ -990,7 +989,7 @@ class FlashCausalLM(Model):
         else:
             x = BLOCK_SIZE // element_size
 
-        if FLASH_DECODING or FLASH_INFER:
+        if ATTENTION in {"flashdecoding", "flashinfer"}:
             self.kv_cache = [
                 (
                     torch.empty(
@@ -1062,7 +1061,7 @@ class FlashCausalLM(Model):
         graph = torch.cuda.CUDAGraph()
         self.cuda_graphs[bs]["graph"] = graph
 
-        if FLASH_INFER:
+        if ATTENTION == "flashinfer":
             from text_generation_server.layers.attention.flash_infer import (
                 create_decode_state_cuda_graphs,
             )
@@ -1766,7 +1765,7 @@ class FlashCausalLM(Model):
         input_lengths: torch.Tensor,
         state: Optional[Any] = None,
     ) -> ContextManager:
-        if not FLASH_INFER:
+        if ATTENTION != "flashinfer":
             return nullcontext()
 
         from text_generation_server.layers.attention.flash_infer import (
diff --git a/server/text_generation_server/models/globals.py b/server/text_generation_server/models/globals.py
index 42b43c87..b58a5b80 100644
--- a/server/text_generation_server/models/globals.py
+++ b/server/text_generation_server/models/globals.py
@@ -5,16 +5,16 @@ from typing import Dict, Optional
 
 from text_generation_server.utils.log import log_master
 
-FLASH_INFER = os.getenv("FLASH_INFER") in {"1", "true", "True"}
-if FLASH_INFER:
-    log_master(logger.info, "Using FLASH_INFER")
+ATTENTION = os.getenv("ATTENTION", "paged")
+_expected = {"paged", "flashdecoding", "flashinfer"}
+assert (
+    ATTENTION in _expected
+), f"Attention is not valid {ATTENTION}, expected {_expected}"
+log_master(logger.info, f"Using Attention = {ATTENTION}")
 
 MEM_POOL = torch.cuda.graph_pool_handle() if torch.cuda.is_available() else None
 # This is overridden by the cli
-FLASH_DECODING = os.getenv("FLASH_DECODING") in {"1", "true", "True"}
-BLOCK_SIZE: int = 256 if FLASH_DECODING else 16
-if FLASH_DECODING:
-    log_master(logger.info, "Using FLASH_DECODING")
+BLOCK_SIZE: int = 256 if ATTENTION == "flashdecoding" else 16
 
 
 cuda_graphs = os.getenv("CUDA_GRAPHS")

From 0d06aed02dba2cfadca5c9fb7e9183545c78f39e Mon Sep 17 00:00:00 2001
From: drbh <david.richard.holtz@gmail.com>
Date: Fri, 9 Aug 2024 10:56:45 -0400
Subject: [PATCH 25/72] feat: add guideline to chat request and template
 (#2391)

* feat: add guideline to chat request and template

* fix: add template test and update docs
---
 docs/openapi.json                 |  7 +++++++
 router/src/infer/chat_template.rs | 15 +++++++++++++++
 router/src/infer/mod.rs           |  3 ++-
 router/src/lib.rs                 |  6 ++++++
 router/src/server.rs              |  9 +++++++--
 5 files changed, 37 insertions(+), 3 deletions(-)

diff --git a/docs/openapi.json b/docs/openapi.json
index ed9b0b96..ecd56e4d 100644
--- a/docs/openapi.json
+++ b/docs/openapi.json
@@ -819,6 +819,13 @@
             "example": "1.0",
             "nullable": true
           },
+          "guideline": {
+            "type": "string",
+            "description": "A guideline to be used in the chat_template",
+            "default": "null",
+            "example": "null",
+            "nullable": true
+          },
           "logit_bias": {
             "type": "array",
             "items": {
diff --git a/router/src/infer/chat_template.rs b/router/src/infer/chat_template.rs
index 24a00352..7c2753ed 100644
--- a/router/src/infer/chat_template.rs
+++ b/router/src/infer/chat_template.rs
@@ -48,6 +48,7 @@ impl ChatTemplate {
 
     pub(crate) fn apply(
         &self,
+        guideline: Option<&str>,
         mut messages: Vec<Message>,
         grammar_with_prompt: Option<(GrammarType, String)>,
     ) -> Result<String, InferError> {
@@ -65,6 +66,7 @@ impl ChatTemplate {
 
         self.template
             .render(ChatTemplateInputs {
+                guideline,
                 messages,
                 bos_token: self.bos_token.as_deref(),
                 eos_token: self.eos_token.as_deref(),
@@ -731,6 +733,19 @@ mod tests {
                 },
                 target: "You are a friendly chatbot who always responds in the style of a pirateYou are a friendly chatbot who always responds in the style of a pirate### Instruction: Hello, how are you?### Response: I'm doing great. How can I help you today?### Instruction: I'd like to show off how chat templating works!",
             },
+            ChatTemplateTestItem {
+                name: "google/shieldgemma-9b",
+                chat_template: "{{- bos_token }}\n{%- if messages[-1].role == 'user' %}\n    {{- \"You are a policy expert trying to help determine whether a user prompt is in violation of the defined safety policies.\\n\\n\" }}\n    {{- \"<start_of_turn>\\n\" }}\n    {{- \"Human Question: \" + messages[-1].content }}\n    {{- \"\\n<end_of_turn>\\n\\n\" }}\n    {{- \"Our safety principle is defined in the below:\\n\\n\" }}\n    {{- \"* \" + guideline + \"\\n\" }}\n    {{- \"\\n===\\n\\n\" }}\n    {{- \"Does the human question violate the above principle? Your answer must start with 'Yes' or 'No'. And then walk through step by step to be sure we answer correctly.\\n\\n\" }}\n{%- else %}\n    {{- \"You are a policy expert trying to help determine whether an AI response to a prompt is in violation of the defined safety policies.\\n\\n\" }}\n    {{- \"<start_of_turn>\\n\" }}\n    {{- \"Human Question: \" + messages[-2].content }}\n    {{- \"\\n<end_of_turn>\\n\" }}\n    {{- \"<start_of_turn>\\n\" }}\n    {{- \"Chatbot Response: \" + messages[-1].content }}\n    {{- \"\\n<end_of_turn>\\n\\n\" }}\n    {{- \"Our safety principle is defined in the below:\\n\\n\" }}\n    {{- \"* \" + guideline + \"\\n\" }}\n    {{- \"\\n===\\n\\n\" }}\n    {{- \"Does the Chatbot Response violate the above principle? Your answer must  start with 'Yes' or 'No'. And then walk through step by step to be sure we answer correctly.\\n\\n\" }}\n{%- endif %}\n\n",
+                input: ChatTemplateInputs {
+                    messages: example_chat_with_system.clone(),
+                    add_generation_prompt: false,
+                    bos_token: Some("<s>"),
+                    eos_token: Some("</s>"),
+                    guideline: Some("Do not use offensive language."),
+                    ..Default::default()
+                },
+                target: "<s>You are a policy expert trying to help determine whether a user prompt is in violation of the defined safety policies.\n\n<start_of_turn>\nHuman Question: I'd like to show off how chat templating works!\n<end_of_turn>\n\nOur safety principle is defined in the below:\n\n* Do not use offensive language.\n\n===\n\nDoes the human question violate the above principle? Your answer must start with 'Yes' or 'No'. And then walk through step by step to be sure we answer correctly.\n\n",
+            },
         ];
 
         #[allow(unused_variables)] // name is unused
diff --git a/router/src/infer/mod.rs b/router/src/infer/mod.rs
index 534a2647..58d5cf9a 100644
--- a/router/src/infer/mod.rs
+++ b/router/src/infer/mod.rs
@@ -138,13 +138,14 @@ impl Infer {
     #[instrument(skip_all)]
     pub(crate) fn apply_chat_template(
         &self,
+        guideline: Option<String>,
         messages: Vec<Message>,
         grammar_with_prompt: Option<(GrammarType, String)>,
     ) -> Result<String, InferError> {
         self.chat_template
             .as_ref()
             .ok_or_else(|| InferError::TemplateError(ErrorKind::TemplateNotFound.into()))?
-            .apply(messages, grammar_with_prompt)
+            .apply(guideline.as_deref(), messages, grammar_with_prompt)
             .map_err(|e| {
                 metrics::counter!("tgi_request_failure", "err" => "template").increment(1);
                 tracing::error!("{e}");
diff --git a/router/src/lib.rs b/router/src/lib.rs
index 66738706..0a15c495 100644
--- a/router/src/lib.rs
+++ b/router/src/lib.rs
@@ -858,6 +858,11 @@ pub(crate) struct ChatRequest {
     #[serde(default)]
     #[schema(nullable = true, default = "null", example = "null")]
     pub response_format: Option<GrammarType>,
+
+    /// A guideline to be used in the chat_template
+    #[serde(default)]
+    #[schema(nullable = true, default = "null", example = "null")]
+    pub guideline: Option<String>,
 }
 
 fn default_tool_prompt() -> Option<String> {
@@ -965,6 +970,7 @@ pub(crate) struct ChatTemplateInputs<'a> {
     add_generation_prompt: bool,
     tools: Option<&'a str>,
     tools_prompt: Option<&'a str>,
+    guideline: Option<&'a str>,
 }
 
 #[derive(Clone, Deserialize, Serialize, ToSchema, Default, Debug, PartialEq)]
diff --git a/router/src/server.rs b/router/src/server.rs
index 1d1cd36a..8c0bd762 100644
--- a/router/src/server.rs
+++ b/router/src/server.rs
@@ -141,6 +141,7 @@ async fn get_chat_tokenize(
         tool_prompt,
         temperature,
         response_format,
+        guideline,
         ..
     } = req;
 
@@ -151,6 +152,7 @@ async fn get_chat_tokenize(
         tools,
         tool_choice,
         &tool_prompt,
+        guideline,
         messages,
     )?;
 
@@ -1123,6 +1125,7 @@ async fn chat_completions(
         tool_prompt,
         temperature,
         response_format,
+        guideline,
         ..
     } = req;
 
@@ -1142,6 +1145,7 @@ async fn chat_completions(
         tools,
         tool_choice,
         &tool_prompt,
+        guideline,
         messages,
     )?;
 
@@ -2402,6 +2406,7 @@ fn prepare_chat_input(
     tools: Option<Vec<Tool>>,
     tool_choice: ToolChoice,
     tool_prompt: &str,
+    guideline: Option<String>,
     messages: Vec<Message>,
 ) -> Result<PreparedInput, InferError> {
     if response_format.is_some() && tools.is_some() {
@@ -2411,7 +2416,7 @@ fn prepare_chat_input(
     }
 
     if let Some(format) = response_format {
-        let inputs = infer.apply_chat_template(messages, None)?;
+        let inputs = infer.apply_chat_template(guideline, messages, None)?;
         return Ok((inputs, Some(format), None));
     }
 
@@ -2423,6 +2428,6 @@ fn prepare_chat_input(
     let tools_grammar_prompt = tool_grammar
         .as_ref()
         .map(|t| (GrammarType::Json(serde_json::json!(t)), tool_prompt.into()));
-    let inputs = infer.apply_chat_template(messages, tools_grammar_prompt)?;
+    let inputs = infer.apply_chat_template(guideline, messages, tools_grammar_prompt)?;
     Ok((inputs, grammar, tool_grammar))
 }

From 8dcc7d3f6ba5dacef1405a2cd1bb8e7b7531070f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Dani=C3=ABl=20de=20Kok?= <me@danieldk.eu>
Date: Fri, 9 Aug 2024 22:36:51 +0200
Subject: [PATCH 26/72] Update flake for 9.0a capability in Torch (#2394)

---
 flake.lock | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/flake.lock b/flake.lock
index 043218e8..7889e7cf 100644
--- a/flake.lock
+++ b/flake.lock
@@ -102,11 +102,11 @@
         "nixpkgs": "nixpkgs"
       },
       "locked": {
-        "lastModified": 1723188417,
-        "narHash": "sha256-GXdFuRMU9N9W0CryUTJIWhJkGwQFHSR2EW5xR0ZyBjk=",
+        "lastModified": 1723234585,
+        "narHash": "sha256-HChJpNP155FPhHr9C5BtqllV8Uv/Ebg59HhMc/HhQrc=",
         "owner": "danieldk",
         "repo": "tgi-nix",
-        "rev": "491db7e234ecf79513ddb94da6ecc14167b9c0b3",
+        "rev": "15bd4a978d6c2c8b04b7afc335d137dbe41e73df",
         "type": "github"
       },
       "original": {

From 01a515dea2f49b0e12a2a4f79e456fc387ab9624 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Dani=C3=ABl=20de=20Kok?= <me@danieldk.eu>
Date: Mon, 12 Aug 2024 09:28:38 +0200
Subject: [PATCH 27/72] nix: add router to the devshell (#2396)

---
 flake.nix  |  4 ++++
 router.nix | 18 ++++++++++++++++++
 2 files changed, 22 insertions(+)
 create mode 100644 router.nix

diff --git a/flake.nix b/flake.nix
index fdd67d00..1d14561f 100644
--- a/flake.nix
+++ b/flake.nix
@@ -70,6 +70,10 @@
                 torch
                 transformers
                 vllm
+
+                (callPackage ./router.nix {
+                  inherit (rustPlatform) buildRustPackage importCargoLock;
+                })
               ]);
 
             venvDir = "./.venv";
diff --git a/router.nix b/router.nix
new file mode 100644
index 00000000..eeeac199
--- /dev/null
+++ b/router.nix
@@ -0,0 +1,18 @@
+{ buildRustPackage, importCargoLock, pkg-config, protobuf, openssl }:
+
+buildRustPackage {
+  name = "text-generation-router";
+
+  src = ./.;
+
+  sourceDir = ./backends/v3;
+
+  cargoLock = {
+    lockFile = ./Cargo.lock;
+  };
+
+  nativeBuildInputs = [ pkg-config ];
+
+  buildInputs = [ openssl.dev protobuf ];
+
+}

From 9c739651cd05ee69daab8d7b99c0c26d082783af Mon Sep 17 00:00:00 2001
From: Nicolas Patry <patry.nicolas@protonmail.com>
Date: Mon, 12 Aug 2024 14:08:38 +0200
Subject: [PATCH 28/72] Upgrade fbgemm (#2398)

* Upgrade fbgemm

* Fix fbgemm version
---
 .../models/test_grammar_response_format_llama.py              | 4 ++--
 server/Makefile-fbgemm                                        | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/integration-tests/models/test_grammar_response_format_llama.py b/integration-tests/models/test_grammar_response_format_llama.py
index ea25fa1c..25bf9d98 100644
--- a/integration-tests/models/test_grammar_response_format_llama.py
+++ b/integration-tests/models/test_grammar_response_format_llama.py
@@ -98,6 +98,6 @@ async def test_grammar_response_format_llama_error_if_tools_not_installed(
     # 422 means the server was unable to process the request because it contains invalid data.
     assert response.status_code == 422
     assert response.json() == {
-        "error": "Grammar and tools are mutually exclusive",
-        "error_type": "grammar and tools",
+        "error": "Tool error: Grammar and tools are mutually exclusive",
+        "error_type": "tool_error",
     }
diff --git a/server/Makefile-fbgemm b/server/Makefile-fbgemm
index 57526577..5f3c0eaa 100644
--- a/server/Makefile-fbgemm
+++ b/server/Makefile-fbgemm
@@ -1,4 +1,4 @@
-fbgemm_commit := ddac8dd9fc0bee70a3f456df68b8aac38576c856
+fbgemm_commit := v0.8.0
 
 build-fbgemm:
 	git clone https://github.com/pytorch/FBGEMM.git fbgemm && \

From 730fa00e2072bccfd61a2c4fd67086128ba76075 Mon Sep 17 00:00:00 2001
From: Nicolas Patry <patry.nicolas@protonmail.com>
Date: Mon, 12 Aug 2024 14:08:46 +0200
Subject: [PATCH 29/72] Adding launcher to build. (#2397)

---
 _launcher.nix | 18 ++++++++++++++++++
 flake.nix     |  3 +++
 2 files changed, 21 insertions(+)
 create mode 100644 _launcher.nix

diff --git a/_launcher.nix b/_launcher.nix
new file mode 100644
index 00000000..1acae7e1
--- /dev/null
+++ b/_launcher.nix
@@ -0,0 +1,18 @@
+{ buildRustPackage, importCargoLock, pkg-config, protobuf, openssl }:
+
+buildRustPackage {
+  name = "text-generation-lancher";
+
+  src = ./.;
+
+  sourceDir = ./launcher;
+
+  cargoLock = {
+    lockFile = ./Cargo.lock;
+  };
+
+  nativeBuildInputs = [ pkg-config ];
+
+  buildInputs = [ openssl.dev protobuf ];
+
+}
diff --git a/flake.nix b/flake.nix
index 1d14561f..761c4af8 100644
--- a/flake.nix
+++ b/flake.nix
@@ -74,6 +74,9 @@
                 (callPackage ./router.nix {
                   inherit (rustPlatform) buildRustPackage importCargoLock;
                 })
+                (callPackage ./_launcher.nix {
+                  inherit (rustPlatform) buildRustPackage importCargoLock;
+                })
               ]);
 
             venvDir = "./.venv";

From 84bc3d7b7d65586f7f249b0e9065588b93e7cab3 Mon Sep 17 00:00:00 2001
From: Nicolas Patry <patry.nicolas@protonmail.com>
Date: Mon, 12 Aug 2024 14:08:59 +0200
Subject: [PATCH 30/72] Fixing import exl2 (#2399)

---
 .../layers/gptq/__init__.py                   | 58 ++++++++++---------
 1 file changed, 30 insertions(+), 28 deletions(-)

diff --git a/server/text_generation_server/layers/gptq/__init__.py b/server/text_generation_server/layers/gptq/__init__.py
index f6616d3e..9c9b69d1 100644
--- a/server/text_generation_server/layers/gptq/__init__.py
+++ b/server/text_generation_server/layers/gptq/__init__.py
@@ -8,34 +8,6 @@ from text_generation_server.utils.import_utils import SYSTEM
 from text_generation_server.utils.log import log_once
 from text_generation_server.utils.weights import Weight, Weights, WeightsLoader
 
-try:
-    major, _minor = torch.cuda.get_device_capability()
-except Exception:
-    major = 1
-
-HAS_EXLLAMA = False
-CAN_EXLLAMA = major >= 8 or SYSTEM == "rocm"
-V2 = os.getenv("EXLLAMA_VERSION", "2") == "2"
-if os.getenv("DISABLE_EXLLAMA") == "True":
-    HAS_EXLLAMA = False
-elif CAN_EXLLAMA:
-    try:
-        if V2:
-            from text_generation_server.layers.gptq.exllamav2 import (
-                QuantLinear as ExllamaQuantLinear,  # noqa: F401
-            )
-
-            HAS_EXLLAMA = "2"
-        else:
-            from text_generation_server.layers.gptq.exllama import (
-                Ex4bitLinear as ExllamaQuantLinear,  # noqa: F401
-            )
-
-            HAS_EXLLAMA = "1"
-
-    except ImportError:
-        pass
-
 
 @dataclass
 class GPTQWeight(Weight):
@@ -432,3 +404,33 @@ class GPTQWeightsLoader(WeightsLoader):
                 else False
             )
             self.quant_method = "gptq"
+
+
+# Needs to be at the end because circular import.
+try:
+    major, _minor = torch.cuda.get_device_capability()
+except Exception:
+    major = 1
+
+HAS_EXLLAMA = False
+CAN_EXLLAMA = major >= 8 or SYSTEM == "rocm"
+V2 = os.getenv("EXLLAMA_VERSION", "2") == "2"
+if os.getenv("DISABLE_EXLLAMA") == "True":
+    HAS_EXLLAMA = False
+elif CAN_EXLLAMA:
+    try:
+        if V2:
+            from text_generation_server.layers.gptq.exllamav2 import (
+                QuantLinear as ExllamaQuantLinear,  # noqa: F401
+            )
+
+            HAS_EXLLAMA = "2"
+        else:
+            from text_generation_server.layers.gptq.exllama import (
+                Ex4bitLinear as ExllamaQuantLinear,  # noqa: F401
+            )
+
+            HAS_EXLLAMA = "1"
+
+    except ImportError:
+        pass

From b6bb1d5160083a011d69c1a32547346a3b4d7d94 Mon Sep 17 00:00:00 2001
From: "Wang, Yi" <yi.a.wang@intel.com>
Date: Mon, 12 Aug 2024 20:10:30 +0800
Subject: [PATCH 31/72] Cpu dockerimage (#2367)

add intel-cpu docker image

Signed-off-by: Wang, Yi A <yi.a.wang@intel.com>
---
 .github/workflows/build.yaml    | 17 +++++++++++++++--
 .github/workflows/ci_build.yaml |  2 +-
 2 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
index 89d5bdf5..fd059e70 100644
--- a/.github/workflows/build.yaml
+++ b/.github/workflows/build.yaml
@@ -50,6 +50,7 @@ jobs:
                 export label_extension=""
                 export docker_devices=""
                 export runs_on="aws-g6-12xlarge-plus-priv"
+                export platform=""
                 ;;
             rocm)
                 export dockerfile="Dockerfile_amd"
@@ -58,12 +59,21 @@ jobs:
                 # TODO Re-enable when they pass.
                 # export runs_on="amd-gpu-tgi"
                 export runs_on="ubuntu-latest"
+                export platform=""
                 ;;
-            intel)
+            intel-xpu)
                 export dockerfile="Dockerfile_intel"
-                export label_extension="-intel"
+                export label_extension="-intel-xpu"
                 export docker_devices=""
                 export runs_on="ubuntu-latest"
+                export platform="xpu"
+                ;;
+            intel-cpu)
+                export dockerfile="Dockerfile_intel"
+                export label_extension="-intel-cpu"
+                export docker_devices=""
+                export runs_on="ubuntu-latest"
+                export platform="cpu"
                 ;;
           esac
           echo $dockerfile
@@ -71,8 +81,10 @@ jobs:
           echo $label_extension
           echo $docker_devices
           echo $runs_on
+          echo $platform
           echo "DOCKERFILE=${dockerfile}" >> $GITHUB_ENV
           echo "LABEL=${label_extension}" >> $GITHUB_ENV
+          echo "PLATFORM=${platform}" >> $GITHUB_ENV
           echo "DOCKER_DEVICES=${docker_devices}" >> $GITHUB_ENV
           echo "RUNS_ON=${runs_on}" >> $GITHUB_ENV
           echo REGISTRY_MIRROR=$REGISTRY_MIRROR >> $GITHUB_ENV
@@ -139,6 +151,7 @@ jobs:
           build-args: |
             GIT_SHA=${{ env.GITHUB_SHA }}
             DOCKER_LABEL=sha-${{ env.GITHUB_SHA_SHORT }}${{ env.LABEL }}
+            PLATFORM=${{ env.PLATFORM }}
           tags: ${{ steps.meta.outputs.tags || steps.meta-pr.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels || steps.meta-pr.outputs.labels }}
           cache-from: type=s3,region=us-east-1,bucket=ci-docker-buildx-cache,name=text-generation-inference-cache${{ env.LABEL }},mode=min,access_key_id=${{ secrets.S3_CI_DOCKER_BUILDX_CACHE_ACCESS_KEY_ID }},secret_access_key=${{ secrets.S3_CI_DOCKER_BUILDX_CACHE_SECRET_ACCESS_KEY }},mode=min
diff --git a/.github/workflows/ci_build.yaml b/.github/workflows/ci_build.yaml
index 5ca2854a..6000cec3 100644
--- a/.github/workflows/ci_build.yaml
+++ b/.github/workflows/ci_build.yaml
@@ -37,7 +37,7 @@ jobs:
       # fail-fast is true by default
       fail-fast: false
       matrix:
-        hardware: ["cuda", "rocm", "intel"]
+        hardware: ["cuda", "rocm", "intel-xpu", "intel-cpu"]
     uses: ./.github/workflows/build.yaml # calls the one above ^
     with:
       hardware: ${{ matrix.hardware }}

From 8deeaca4ff2080381e4ed00c98e1711d896687ad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Dani=C3=ABl=20de=20Kok?= <me@danieldk.eu>
Date: Mon, 12 Aug 2024 14:59:17 +0200
Subject: [PATCH 32/72] Add support for prefix caching to the v3 router (#2392)

This change adds support for prefix caching to the v3 router. This
is broken up from the backend support to ease reviewing.

For now prefix caching is only enabled with `USE_PREFIX_CACHING=1`
in this case, the router will switch to `RadixAllocator`. This
allocator uses a radix trie to keep track of prefills that were
seen prior. If a new prefill is a prefix of a previously-seen
prefil, the router will send a request with `prefix_len>0`, which
can be used by the backend to decide to reuse KV blocks from the
cache, rather than recomputing them.

Even though backend support is not added in this PR, the backend
will still work with prefix caching enabled. The prefix lengths
are just ignored and not used.
---
 Cargo.lock                                    |   1 +
 backends/client/src/v3/client.rs              |   1 +
 backends/client/src/v3/sharded_client.rs      |   1 +
 backends/v3/Cargo.toml                        |   1 +
 backends/v3/src/backend.rs                    |  10 +
 backends/v3/src/block_allocator.rs            | 182 +++--
 backends/v3/src/client/grpc_client.rs         |   1 +
 backends/v3/src/client/sharded_client.rs      |   1 +
 backends/v3/src/lib.rs                        |   1 +
 backends/v3/src/queue.rs                      |  53 +-
 backends/v3/src/radix.rs                      | 755 ++++++++++++++++++
 benchmark/src/generation.rs                   |   1 +
 proto/v3/generate.proto                       | 356 ++++-----
 router/src/validation.rs                      |  19 +-
 .../text_generation_server/models/globals.py  |  17 +-
 15 files changed, 1145 insertions(+), 255 deletions(-)
 create mode 100644 backends/v3/src/radix.rs

diff --git a/Cargo.lock b/Cargo.lock
index 92367d1e..3a5845a7 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4045,6 +4045,7 @@ dependencies = [
  "reqwest",
  "serde",
  "serde_json",
+ "slotmap",
  "text-generation-router",
  "thiserror",
  "tokenizers",
diff --git a/backends/client/src/v3/client.rs b/backends/client/src/v3/client.rs
index a996b14f..b321278c 100644
--- a/backends/client/src/v3/client.rs
+++ b/backends/client/src/v3/client.rs
@@ -156,6 +156,7 @@ impl Client {
                 // Blocks and slots will be set on the server side if we use paged attention
                 blocks: vec![],
                 slots: vec![],
+                prefix_len: 0,
                 // Set sampling parameters to also take these ops into account in the max memory
                 parameters: Some(NextTokenChooserParameters {
                     temperature: 0.9,
diff --git a/backends/client/src/v3/sharded_client.rs b/backends/client/src/v3/sharded_client.rs
index ae8a899b..1cc173e3 100644
--- a/backends/client/src/v3/sharded_client.rs
+++ b/backends/client/src/v3/sharded_client.rs
@@ -244,6 +244,7 @@ impl Health for ShardedClient {
             // Block 0 is reserved for health checks
             blocks: vec![0],
             slots: (0..16).collect(),
+            prefix_len: 0,
             adapter_id: None,
         };
         let batch = Batch {
diff --git a/backends/v3/Cargo.toml b/backends/v3/Cargo.toml
index 5d9a140b..129ceb9c 100644
--- a/backends/v3/Cargo.toml
+++ b/backends/v3/Cargo.toml
@@ -33,6 +33,7 @@ rand = "0.8.5"
 reqwest = { version = "0.11.20", features = [] }
 serde = "1.0.188"
 serde_json = "1.0.107"
+slotmap = "1.0.7"
 thiserror = "1.0.48"
 tokenizers = { workspace = true}
 tokio = { version = "1.32.0", features = ["rt", "rt-multi-thread", "parking_lot", "signal", "sync"] }
diff --git a/backends/v3/src/backend.rs b/backends/v3/src/backend.rs
index 68ddf00b..cbcbff72 100644
--- a/backends/v3/src/backend.rs
+++ b/backends/v3/src/backend.rs
@@ -35,15 +35,24 @@ impl BackendV3 {
         window_size: Option<u32>,
         speculate: u32,
     ) -> Self {
+        let prefix_caching = if let Ok(prefix_caching) = std::env::var("USE_PREFIX_CACHING") {
+            matches!(prefix_caching.as_str(), "true" | "1")
+        } else {
+            false
+        };
         let attention = if let Ok(attention) = std::env::var("ATTENTION") {
             attention
                 .parse()
                 .unwrap_or_else(|_| panic!("Invalid attention was specified :`{attention}`"))
+        } else if prefix_caching {
+            Attention::FlashInfer
         } else {
             Attention::Paged
         };
         let block_size = if attention == Attention::FlashDecoding {
             256
+        } else if attention == Attention::FlashInfer {
+            1
         } else {
             16
         };
@@ -51,6 +60,7 @@ impl BackendV3 {
         let queue = Queue::new(
             requires_padding,
             block_size,
+            prefix_caching,
             window_size,
             speculate,
             max_batch_total_tokens,
diff --git a/backends/v3/src/block_allocator.rs b/backends/v3/src/block_allocator.rs
index 7467fd85..05c2bd30 100644
--- a/backends/v3/src/block_allocator.rs
+++ b/backends/v3/src/block_allocator.rs
@@ -1,16 +1,26 @@
-use std::cmp::min;
+use std::{cmp::min, sync::Arc};
 use tokio::sync::{mpsc, oneshot};
 
+use crate::radix::RadixAllocator;
+
 #[derive(Debug, Clone)]
 pub(crate) struct BlockAllocation {
+    pub allocation_id: u64,
     pub blocks: Vec<u32>,
     pub slots: Vec<u32>,
-    block_allocator: BlockAllocator,
+
+    /// Prefix that was cached and for which the KV does not have to
+    /// be recomputed.
+    pub prefix_len: u32,
+
+    pub(crate) block_allocator: Option<BlockAllocator>,
 }
 
 impl Drop for BlockAllocation {
     fn drop(&mut self) {
-        self.block_allocator.free(self.blocks.clone())
+        if let Some(block_allocator) = self.block_allocator.as_mut() {
+            block_allocator.free(self.blocks.clone(), self.allocation_id)
+        }
     }
 }
 
@@ -24,6 +34,7 @@ impl BlockAllocator {
     pub(crate) fn new(
         max_batch_total_tokens: u32,
         block_size: u32,
+        prefix_caching: bool,
         window_size: Option<u32>,
     ) -> Self {
         // Create channel
@@ -33,6 +44,7 @@ impl BlockAllocator {
         tokio::spawn(block_allocator_task(
             max_batch_total_tokens / block_size,
             block_size,
+            prefix_caching,
             window_size,
             receiver,
         ));
@@ -42,28 +54,32 @@ impl BlockAllocator {
         }
     }
 
-    pub(crate) async fn allocate(&self, tokens: u32) -> Option<BlockAllocation> {
+    pub(crate) async fn allocate(
+        &self,
+        tokens: u32,
+        prefill_tokens: Option<Arc<Vec<u32>>>,
+    ) -> Option<BlockAllocation> {
         let (response_sender, response_receiver) = oneshot::channel();
         self.block_allocator
             .send(BlockAllocatorCommand::Allocate {
                 tokens,
+                prefill_tokens,
                 response_sender,
             })
             .unwrap();
 
-        response_receiver
-            .await
-            .unwrap()
-            .map(|(blocks, slots)| BlockAllocation {
-                blocks,
-                slots,
-                block_allocator: self.clone(),
-            })
+        response_receiver.await.unwrap().map(|mut allocation| {
+            allocation.block_allocator = Some(self.clone());
+            allocation
+        })
     }
 
-    pub(crate) fn free(&self, blocks: Vec<u32>) {
+    pub(crate) fn free(&self, blocks: Vec<u32>, allocation_id: u64) {
         self.block_allocator
-            .send(BlockAllocatorCommand::Free { blocks })
+            .send(BlockAllocatorCommand::Free {
+                allocation_id,
+                blocks,
+            })
             .unwrap();
     }
 }
@@ -71,54 +87,29 @@ impl BlockAllocator {
 async fn block_allocator_task(
     blocks: u32,
     block_size: u32,
+    prefix_caching: bool,
     window_size: Option<u32>,
     mut receiver: mpsc::UnboundedReceiver<BlockAllocatorCommand>,
 ) {
-    // Block 0 is reserved for health checks
-    let mut free_blocks: Vec<u32> = (1..blocks).collect();
+    let mut allocator: Box<dyn Allocator + Send> = if prefix_caching {
+        Box::new(RadixAllocator::new(block_size, blocks, window_size))
+    } else {
+        Box::new(SimpleAllocator::new(blocks, block_size, window_size))
+    };
     while let Some(cmd) = receiver.recv().await {
         match cmd {
-            BlockAllocatorCommand::Free { blocks } => free_blocks.extend(blocks),
+            BlockAllocatorCommand::Free {
+                blocks,
+                allocation_id,
+            } => allocator.free(blocks, allocation_id),
             BlockAllocatorCommand::Allocate {
                 tokens,
+                prefill_tokens,
                 response_sender,
             } => {
-                // Apply window size
-                let (required_blocks, repeats) = {
-                    let (tokens, repeats) = match window_size {
-                        None => (tokens, 1),
-                        Some(window_size) => {
-                            let repeats = (tokens + window_size - 1) / window_size;
-                            let tokens = min(tokens, window_size);
-                            (tokens, repeats as usize)
-                        }
-                    };
-                    // Pad to a multiple of block size
-                    let required_blocks = (tokens + block_size - 1) / block_size;
-                    (required_blocks, repeats)
-                };
-
-                let tokens = tokens as usize;
-                let allocation = if required_blocks > free_blocks.len() as u32 {
-                    None
-                } else {
-                    let blocks =
-                        free_blocks.split_off(free_blocks.len() - required_blocks as usize);
-                    let mut slots = Vec::with_capacity(
-                        (required_blocks * block_size * repeats as u32) as usize,
-                    );
-
-                    'slots: for block_id in blocks.repeat(repeats).iter() {
-                        for s in (block_id * block_size)..((block_id + 1) * block_size) {
-                            slots.push(s);
-                            if slots.len() == tokens {
-                                break 'slots;
-                            }
-                        }
-                    }
-                    Some((blocks, slots))
-                };
-                response_sender.send(allocation).unwrap();
+                response_sender
+                    .send(allocator.allocate(tokens, prefill_tokens))
+                    .unwrap();
             }
         }
     }
@@ -128,9 +119,92 @@ async fn block_allocator_task(
 enum BlockAllocatorCommand {
     Free {
         blocks: Vec<u32>,
+        allocation_id: u64,
     },
     Allocate {
         tokens: u32,
-        response_sender: oneshot::Sender<Option<(Vec<u32>, Vec<u32>)>>,
+        prefill_tokens: Option<Arc<Vec<u32>>>,
+        response_sender: oneshot::Sender<Option<BlockAllocation>>,
     },
 }
+
+pub(crate) trait Allocator {
+    fn allocate(
+        &mut self,
+        tokens: u32,
+        prefill_tokens: Option<Arc<Vec<u32>>>,
+    ) -> Option<BlockAllocation>;
+
+    fn free(&mut self, blocks: Vec<u32>, allocation_id: u64);
+}
+
+pub struct SimpleAllocator {
+    free_blocks: Vec<u32>,
+    block_size: u32,
+    window_size: Option<u32>,
+}
+
+impl SimpleAllocator {
+    fn new(blocks: u32, block_size: u32, window_size: Option<u32>) -> Self {
+        SimpleAllocator {
+            block_size,
+            // Block 0 is reserved for health checks
+            free_blocks: (1..blocks).collect(),
+            window_size,
+        }
+    }
+}
+
+impl Allocator for SimpleAllocator {
+    fn allocate(
+        &mut self,
+        tokens: u32,
+        _prefill_tokens: Option<Arc<Vec<u32>>>,
+    ) -> Option<BlockAllocation> {
+        // Apply window size
+        let (required_blocks, repeats) = {
+            let (tokens, repeats) = match self.window_size {
+                None => (tokens, 1),
+                Some(window_size) => {
+                    let repeats = (tokens + window_size - 1) / window_size;
+                    let tokens = min(tokens, window_size);
+                    (tokens, repeats as usize)
+                }
+            };
+            // Pad to a multiple of block size
+            let required_blocks = (tokens + self.block_size - 1) / self.block_size;
+            (required_blocks, repeats)
+        };
+
+        let tokens = tokens as usize;
+        if required_blocks > self.free_blocks.len() as u32 {
+            None
+        } else {
+            let blocks = self
+                .free_blocks
+                .split_off(self.free_blocks.len() - required_blocks as usize);
+            let mut slots =
+                Vec::with_capacity((required_blocks * self.block_size * repeats as u32) as usize);
+
+            'slots: for block_id in blocks.repeat(repeats).iter() {
+                for s in (block_id * self.block_size)..((block_id + 1) * self.block_size) {
+                    slots.push(s);
+                    if slots.len() == tokens {
+                        break 'slots;
+                    }
+                }
+            }
+            Some(BlockAllocation {
+                allocation_id: 0,
+                blocks,
+                slots,
+                prefix_len: 0,
+                block_allocator: None,
+            })
+        }
+    }
+
+    fn free(&mut self, blocks: Vec<u32>, _allocation_id: u64) {
+        self.free_blocks.extend(blocks)
+    }
+}
diff --git a/backends/v3/src/client/grpc_client.rs b/backends/v3/src/client/grpc_client.rs
index c407687b..6282759e 100644
--- a/backends/v3/src/client/grpc_client.rs
+++ b/backends/v3/src/client/grpc_client.rs
@@ -157,6 +157,7 @@ impl Client {
                 // Blocks and slots will be set on the server side if we use paged attention
                 blocks: vec![],
                 slots: vec![],
+                prefix_len: 0,
                 // Set sampling parameters to also take these ops into account in the max memory
                 parameters: Some(NextTokenChooserParameters {
                     temperature: 0.9,
diff --git a/backends/v3/src/client/sharded_client.rs b/backends/v3/src/client/sharded_client.rs
index afb13cdc..2f78da03 100644
--- a/backends/v3/src/client/sharded_client.rs
+++ b/backends/v3/src/client/sharded_client.rs
@@ -245,6 +245,7 @@ impl Health for ShardedClient {
             // Block 0 is reserved for health checks
             blocks: vec![0],
             slots: (0..16).collect(),
+            prefix_len: 0,
             adapter_id: None,
         };
         let batch = Batch {
diff --git a/backends/v3/src/lib.rs b/backends/v3/src/lib.rs
index a6f89169..c8fc55f8 100644
--- a/backends/v3/src/lib.rs
+++ b/backends/v3/src/lib.rs
@@ -2,6 +2,7 @@ mod backend;
 mod block_allocator;
 mod client;
 mod queue;
+mod radix;
 
 use crate::client::{ClientError, ShardedClient};
 pub(crate) use backend::BackendV3;
diff --git a/backends/v3/src/queue.rs b/backends/v3/src/queue.rs
index b457389c..13544235 100644
--- a/backends/v3/src/queue.rs
+++ b/backends/v3/src/queue.rs
@@ -46,6 +46,7 @@ impl Queue {
     pub(crate) fn new(
         requires_padding: bool,
         block_size: u32,
+        prefix_caching: bool,
         window_size: Option<u32>,
         speculate: u32,
         max_batch_total_tokens: u32,
@@ -57,6 +58,7 @@ impl Queue {
         tokio::spawn(queue_task(
             requires_padding,
             block_size,
+            prefix_caching,
             window_size,
             speculate,
             max_batch_total_tokens,
@@ -109,6 +111,7 @@ impl Queue {
 async fn queue_task(
     requires_padding: bool,
     block_size: u32,
+    prefix_caching: bool,
     window_size: Option<u32>,
     speculate: u32,
     max_batch_total_tokens: u32,
@@ -117,6 +120,7 @@ async fn queue_task(
     let mut state = State::new(
         requires_padding,
         block_size,
+        prefix_caching,
         window_size,
         speculate,
         max_batch_total_tokens,
@@ -176,12 +180,19 @@ impl State {
     fn new(
         requires_padding: bool,
         block_size: u32,
+        prefix_caching: bool,
         window_size: Option<u32>,
         speculate: u32,
         max_batch_total_tokens: u32,
     ) -> Self {
-        let block_allocator = (!requires_padding)
-            .then(|| BlockAllocator::new(max_batch_total_tokens, block_size, window_size));
+        let block_allocator = (!requires_padding).then(|| {
+            BlockAllocator::new(
+                max_batch_total_tokens,
+                block_size,
+                prefix_caching,
+                window_size,
+            )
+        });
 
         Self {
             entries: VecDeque::with_capacity(128),
@@ -305,7 +316,10 @@ impl State {
                         + self.speculate
                         - 1;
 
-                    match block_allocator.allocate(tokens).await {
+                    match block_allocator
+                        .allocate(tokens, entry.request.input_ids.clone())
+                        .await
+                    {
                         None => {
                             // Entry is over budget
                             // Add it back to the front
@@ -331,11 +345,12 @@ impl State {
             // Update entry
             entry.temp_span = Some(entry_batch_span);
 
-            let (blocks, slots) = match &block_allocation {
-                None => (Vec::new(), Vec::new()),
+            let (blocks, slots, prefix_len) = match &block_allocation {
+                None => (Vec::new(), Vec::new(), 0),
                 Some(block_allocation) => (
                     block_allocation.blocks.clone(),
                     block_allocation.slots.clone(),
+                    block_allocation.prefix_len,
                 ),
             };
 
@@ -372,6 +387,7 @@ impl State {
                 top_n_tokens: entry.request.top_n_tokens,
                 blocks,
                 slots,
+                prefix_len,
                 adapter_id: entry.request.adapter_id.clone(),
             });
             // Set batch_time
@@ -480,6 +496,8 @@ impl From<ValidStoppingParameters> for StoppingCriteriaParameters {
 
 #[cfg(test)]
 mod tests {
+    use std::sync::Arc;
+
     use super::*;
     use tracing::info_span;
 
@@ -492,6 +510,7 @@ mod tests {
         let entry = Entry {
             request: ValidGenerateRequest {
                 inputs: vec![],
+                input_ids: Some(Arc::new(vec![])),
                 input_length: 0,
                 truncate: 0,
                 decoder_input_details: false,
@@ -527,7 +546,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_append() {
-        let mut state = State::new(false, 1, None, 0, 16);
+        let mut state = State::new(false, 1, false, None, 0, 16);
         let (entry, _guard) = default_entry();
 
         assert_eq!(state.next_id, 0);
@@ -543,7 +562,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_next_batch_empty() {
-        let mut state = State::new(false, 1, None, 0, 16);
+        let mut state = State::new(false, 1, false, None, 0, 16);
 
         assert!(state.next_batch(None, None, 1, 1).await.is_none());
         assert!(state.next_batch(Some(1), None, 1, 1).await.is_none());
@@ -551,7 +570,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_next_batch_min_size() {
-        let mut state = State::new(false, 1, None, 0, 16);
+        let mut state = State::new(false, 1, false, None, 0, 16);
         let (entry1, _guard1) = default_entry();
         let (entry2, _guard2) = default_entry();
         state.append(entry1);
@@ -583,7 +602,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_next_batch_max_size() {
-        let mut state = State::new(false, 1, None, 0, 16);
+        let mut state = State::new(false, 1, false, None, 0, 16);
         let (entry1, _guard1) = default_entry();
         let (entry2, _guard2) = default_entry();
         state.append(entry1);
@@ -603,7 +622,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_next_batch_token_budget() {
-        let mut state = State::new(false, 1, None, 0, 2);
+        let mut state = State::new(false, 1, false, None, 0, 2);
         let (entry1, _guard1) = default_entry();
         let (entry2, _guard2) = default_entry();
         state.append(entry1);
@@ -636,14 +655,14 @@ mod tests {
 
     #[tokio::test]
     async fn test_queue_append() {
-        let queue = Queue::new(false, 1, None, 0, 16);
+        let queue = Queue::new(false, 1, false, None, 0, 16);
         let (entry, _guard) = default_entry();
         queue.append(entry);
     }
 
     #[tokio::test]
     async fn test_queue_next_batch_empty() {
-        let queue = Queue::new(false, 1, None, 0, 16);
+        let queue = Queue::new(false, 1, false, None, 0, 16);
 
         assert!(queue.next_batch(None, None, 1, 1).await.is_none());
         assert!(queue.next_batch(Some(1), None, 1, 1).await.is_none());
@@ -651,7 +670,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_queue_next_batch_min_size() {
-        let queue = Queue::new(false, 1, None, 0, 16);
+        let queue = Queue::new(false, 1, false, None, 0, 16);
         let (entry1, _guard1) = default_entry();
         let (entry2, _guard2) = default_entry();
         queue.append(entry1);
@@ -684,7 +703,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_queue_next_batch_max_size() {
-        let queue = Queue::new(false, 1, None, 0, 16);
+        let queue = Queue::new(false, 1, false, None, 0, 16);
         let (entry1, _guard1) = default_entry();
         let (entry2, _guard2) = default_entry();
         queue.append(entry1);
@@ -700,7 +719,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_queue_next_batch_token_budget() {
-        let queue = Queue::new(false, 1, None, 0, 16);
+        let queue = Queue::new(false, 1, false, None, 0, 16);
         let (entry1, _guard1) = default_entry();
         let (entry2, _guard2) = default_entry();
         queue.append(entry1);
@@ -725,7 +744,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_queue_next_batch_token_speculate() {
-        let queue = Queue::new(false, 1, None, 2, 16);
+        let queue = Queue::new(false, 1, false, None, 2, 16);
         let (entry1, _guard1) = default_entry();
         let (entry2, _guard2) = default_entry();
         queue.append(entry1);
@@ -744,7 +763,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_queue_next_batch_dropped_receiver() {
-        let queue = Queue::new(false, 1, None, 0, 16);
+        let queue = Queue::new(false, 1, false, None, 0, 16);
         let (entry, _) = default_entry();
         queue.append(entry);
 
diff --git a/backends/v3/src/radix.rs b/backends/v3/src/radix.rs
new file mode 100644
index 00000000..0464b9f8
--- /dev/null
+++ b/backends/v3/src/radix.rs
@@ -0,0 +1,755 @@
+use std::{
+    collections::{BTreeSet, HashMap},
+    sync::Arc,
+};
+
+use slotmap::{DefaultKey, SlotMap};
+
+use crate::block_allocator::{Allocator, BlockAllocation};
+
+pub struct RadixAllocator {
+    allocation_id: u64,
+
+    allocations: HashMap<u64, RadixAllocation>,
+
+    cache_blocks: RadixTrie,
+
+    /// Blocks that are immediately available for allocation.
+    free_blocks: Vec<u32>,
+}
+
+impl RadixAllocator {
+    pub fn new(block_size: u32, n_blocks: u32, window_size: Option<u32>) -> Self {
+        assert_eq!(
+            block_size, 1,
+            "Radix tree allocator only works with block_size=1, was: {}",
+            block_size
+        );
+        if window_size.is_some() {
+            unimplemented!("Window size not supported in the prefix-caching block allocator yet");
+        }
+
+        RadixAllocator {
+            allocation_id: 0,
+            allocations: HashMap::new(),
+            cache_blocks: RadixTrie::new(),
+
+            // Block 0 is reserved for health checks.
+            free_blocks: (1..n_blocks).collect(),
+        }
+    }
+
+    fn alloc_or_reclaim(&mut self, n_blocks_needed: usize) -> Option<Vec<u32>> {
+        if self.free_blocks.len() < n_blocks_needed {
+            // This is a bit annoying, we first extend the free list and then
+            // split it off again below. This is because we need to put it on
+            // the free list if we cannot allocate enough blocks. This is only
+            // temporary, the trie needs to be able to report whether it can
+            // allocate the requested amount. Just not implemented yet.
+            self.free_blocks.extend(
+                self.cache_blocks
+                    .evict(n_blocks_needed - self.free_blocks.len()),
+            );
+        }
+
+        if self.free_blocks.len() >= n_blocks_needed {
+            Some(
+                self.free_blocks
+                    .split_off(self.free_blocks.len() - n_blocks_needed),
+            )
+        } else {
+            None
+        }
+    }
+}
+
+impl Allocator for RadixAllocator {
+    fn allocate(
+        &mut self,
+        tokens: u32,
+        prefill_tokens: Option<Arc<Vec<u32>>>,
+    ) -> Option<BlockAllocation> {
+        let mut blocks = vec![];
+        let prefix_node = if let Some(prefill_tokens) = prefill_tokens.as_ref() {
+            let node_id = self
+                .cache_blocks
+                .find(prefill_tokens.as_slice(), &mut blocks);
+            // Even if this allocation fails below, we need to increase he
+            // refcount to ensure that the prefix that was found is not evicted.
+
+            node_id
+        } else {
+            self.cache_blocks.root_id()
+        };
+
+        self.cache_blocks
+            .incref(prefix_node)
+            .expect("Failed to increment refcount");
+
+        let prefix_len = blocks.len();
+        let suffix_len = tokens - prefix_len as u32;
+
+        match self.alloc_or_reclaim(suffix_len as usize) {
+            Some(suffix_blocks) => blocks.extend(suffix_blocks),
+            None => {
+                self.cache_blocks
+                    .decref(prefix_node)
+                    .expect("Failed to decrement refcount");
+                return None;
+            }
+        }
+
+        // 1:1 mapping of blocks and slots.
+        let slots = blocks.clone();
+
+        let allocation = RadixAllocation {
+            prefix_node,
+            cached_prefix_len: prefix_len,
+            prefill_tokens: prefill_tokens.clone(),
+        };
+
+        self.allocation_id += 1;
+        self.allocations.insert(self.allocation_id, allocation);
+
+        Some(BlockAllocation {
+            allocation_id: self.allocation_id,
+            block_allocator: None,
+            blocks,
+            slots,
+            prefix_len: prefix_len as u32,
+        })
+    }
+
+    fn free(&mut self, blocks: Vec<u32>, allocation_id: u64) {
+        let allocation = match self.allocations.remove(&allocation_id) {
+            Some(allocation) => allocation,
+            None => unreachable!("Tried to free an unknown allocation."),
+        };
+
+        self.cache_blocks
+            .decref(allocation.prefix_node)
+            .expect("Failed to decrement refcount");
+
+        if let Some(prefill_tokens) = allocation.prefill_tokens {
+            let prefill_tokens = prefill_tokens.as_slice();
+
+            // If there are prefill tokens that did not come from the cache,
+            // add them to the cache.
+            if prefill_tokens.len() > allocation.cached_prefix_len {
+                let prefix_len = self
+                    .cache_blocks
+                    .insert(prefill_tokens, &blocks[..prefill_tokens.len()])
+                    // Unwrap, failing is a programming error.
+                    .expect("Failed to store prefill tokens");
+
+                // We can have a prefill with the following structure:
+                //
+                // |---| From the prefix cache.
+                // A B C D E F G
+                //|--------| Found in the trie during insertion.
+                //
+                // This means that while processing this request there was a
+                // partially overlapping request that had A..=E in its
+                // prefill. In this case we need to free the blocks D E.
+                self.free_blocks
+                    .extend(&blocks[allocation.cached_prefix_len..prefix_len]);
+            }
+
+            // Free non-prefill blocks.
+            self.free_blocks.extend(&blocks[prefill_tokens.len()..]);
+        } else {
+            self.free_blocks.extend(blocks);
+        }
+    }
+}
+
+struct RadixAllocation {
+    prefix_node: NodeId,
+    cached_prefix_len: usize,
+    prefill_tokens: Option<Arc<Vec<u32>>>,
+}
+
+// Radix trie that is heavily inspired by radix attention from sglang.
+//
+// The trie is optimized for prefix caching:
+//
+// - A normal radix trie stores discrete values. In this radix trie,
+//   inserting *abc* with value *xyz* will also enable lookup for
+//   *a* (*x*) and *ab* (*xy*).
+// - As a result, every value is required to have the same length as
+//   the key.
+// - We store additional information in each node, such as last access
+//   time and a reference count.
+
+#[derive(Debug)]
+pub enum TrieError {
+    InvalidNodeId,
+    RefCountUnderflow,
+    BlockTokenCountMismatch,
+}
+
+pub type NodeId = DefaultKey;
+
+#[derive(Debug)]
+pub struct RadixTrie {
+    /// Identifier of the root nod.
+    root: DefaultKey,
+
+    /// Leave node identifiers ordered by increasing recency.
+    leaves: BTreeSet<(u64, NodeId)>,
+
+    /// All trie nodes.
+    nodes: SlotMap<NodeId, TrieNode>,
+
+    /// Time as a monotonically increating counter to avoid the system
+    /// call that a real time lookup would require.
+    time: u64,
+}
+
+impl RadixTrie {
+    /// Construct a new radix trie.
+    pub fn new() -> Self {
+        let root = TrieNode::new(vec![], vec![], 0, None);
+        let mut nodes = SlotMap::new();
+        let root = nodes.insert(root);
+        RadixTrie {
+            leaves: BTreeSet::new(),
+            nodes,
+            root,
+            time: 0,
+        }
+    }
+
+    /// Find the prefix of the given tokens.
+    ///
+    /// The blocks corresponding to the part of the prefix that could be found
+    /// are writteng to `blocks`. The number of blocks is in `0..=tokens.len()`.
+    /// Returns the identifier of the trie node that contains the longest
+    /// prefix. The node identifier can be used by callers to e.g. increase its
+    /// reference count.
+    ///
+    /// Using this method will update the access time of the traversed nodes.
+    pub fn find(&mut self, key: &[u32], blocks: &mut Vec<u32>) -> NodeId {
+        self.time += 1;
+        self.find_(self.root, key, blocks)
+    }
+
+    /// Find worker.
+    fn find_(&mut self, mut node_id: NodeId, key: &[u32], blocks: &mut Vec<u32>) -> NodeId {
+        let node = &self.nodes[node_id];
+
+        if let Some(&child_id) = node.children.get(&key[0]) {
+            self.update_access_time(child_id);
+            let child = self.nodes.get(child_id).expect("Invalid child identifier");
+            let shared_prefix_len = child.key.shared_prefix_len(key);
+            blocks.extend(&child.blocks[..shared_prefix_len]);
+
+            let key = &key[shared_prefix_len..];
+            if !key.is_empty() {
+                node_id = self.find_(child_id, key, blocks);
+            }
+        }
+
+        node_id
+    }
+
+    /// Decrease the reference count of a node.
+    pub fn decref(&mut self, node_id: NodeId) -> Result<(), TrieError> {
+        // We don't care about refcounting for root, since it will never
+        // be evicted.
+        if node_id == self.root {
+            return Ok(());
+        }
+
+        let node = self
+            .nodes
+            .get_mut(node_id)
+            .ok_or(TrieError::InvalidNodeId)?;
+        if node.ref_count == 0 {
+            return Err(TrieError::RefCountUnderflow);
+        }
+
+        node.ref_count -= 1;
+        if node.ref_count == 0 {
+            self.leaves.insert((node.last_accessed, node_id));
+        }
+
+        Ok(())
+    }
+
+    /// Increase the reference count of a node.
+    pub fn incref(&mut self, node_id: NodeId) -> Result<(), TrieError> {
+        if node_id == self.root {
+            return Ok(());
+        }
+
+        let node = self
+            .nodes
+            .get_mut(node_id)
+            .ok_or(TrieError::InvalidNodeId)?;
+        if node.ref_count == 0 {
+            self.leaves.remove(&(node.last_accessed, node_id));
+        }
+        node.ref_count += 1;
+
+        Ok(())
+    }
+
+    /// Evict `n_blocks` from the trie.
+    ///
+    /// Returns the evicted blocks. When the length is less than `n_blocks`,
+    /// not enough blocks could beevicted.
+    pub fn evict(&mut self, n_blocks: usize) -> Vec<u32> {
+        // NOTE: we don't return Result here. If any of the unwrapping fails,
+        // it's a programming error in the trie implementation, not a user
+        // error caused by e.g. an invalid argument.
+
+        // TODO: add some bookkeeping in the future to check whether we can
+        // evict n_blocks and return `None` if we can't. We are now needlessly
+        // evicting prefixes from the cache in such a case.
+        let mut evicted = Vec::new();
+
+        while let Some((last_access, node_id)) = self.leaves.pop_first() {
+            let blocks_needed = n_blocks - evicted.len();
+
+            let node = self.nodes.get(node_id).expect("Leave does not exist");
+            if blocks_needed >= node.blocks.len() {
+                // We need to evict the whole node if we need more blocks than it has.
+                let node = self.remove_node(node_id);
+                evicted.extend(node.blocks);
+
+                if evicted.len() >= n_blocks {
+                    break;
+                }
+            } else {
+                // The node has more blocks than needed, so we'll just remove
+                // the required number of blocks and leave the remaining blocks
+                // untouched.
+                let node = self.nodes.get_mut(node_id).expect("Leave does not exist");
+                node.key.truncate(node.blocks.len() - blocks_needed);
+                evicted.extend(node.blocks.split_off(node.blocks.len() - blocks_needed));
+                self.leaves.insert((last_access, node_id));
+                break;
+            }
+        }
+
+        evicted
+    }
+
+    /// Insert a prefill along with its blocks.
+    ///
+    /// This method returns the length of the prefix that was already
+    /// in the trie. E.g. if the length is 10, this means that for
+    /// the first 10 elements of the tree **the blocks are not updated**.
+    pub fn insert(&mut self, tokens: &[u32], blocks: &[u32]) -> Result<usize, TrieError> {
+        self.time += 1;
+        self.insert_(self.root, tokens, blocks)
+    }
+
+    /// Insertion worker.
+    fn insert_(
+        &mut self,
+        node_id: NodeId,
+        tokens: &[u32],
+        blocks: &[u32],
+    ) -> Result<usize, TrieError> {
+        // TODO: in the future we may want to check that the blocks match for
+        // the part of the prefix that is already in the trie to detect
+        // mismatches.
+
+        if tokens.len() != blocks.len() {
+            return Err(TrieError::BlockTokenCountMismatch);
+        }
+
+        if let Some(&child_id) = self.nodes[node_id].children.get(&tokens[0]) {
+            self.update_access_time(child_id);
+            let child = self
+                .nodes
+                .get_mut(child_id)
+                // Unwrap here, since failure is a bug.
+                .expect("Child node does not exist");
+            let shared_prefix_len = child.key.shared_prefix_len(tokens);
+
+            // We are done, the prefix is already in the trie.
+            if shared_prefix_len == tokens.len() {
+                return Ok(shared_prefix_len);
+            }
+
+            // The node's prefix is a prefix of the insertion prefix.
+            if shared_prefix_len == child.key.len() {
+                return Ok(shared_prefix_len
+                    + self.insert_(
+                        child_id,
+                        &tokens[shared_prefix_len..],
+                        &blocks[shared_prefix_len..],
+                    )?);
+            }
+
+            // The node's prefix and the insertion prefix only match partially,
+            // split the node to just contain the matching part. Then insert the
+            // remainder of the prefix into the node again
+            let child_id = self.split_node(child_id, shared_prefix_len);
+            let key = &tokens[shared_prefix_len..];
+            let blocks = &blocks[shared_prefix_len..];
+            Ok(shared_prefix_len + self.insert_(child_id, key, blocks)?)
+        } else {
+            self.add_node(node_id, tokens, blocks);
+            Ok(0)
+        }
+    }
+
+    fn split_node(&mut self, node_id: NodeId, prefix_len: usize) -> NodeId {
+        // We have to make the current node a child to ensure that its
+        // properties and node id stay the same.
+
+        // This funcion unwraps, an  invalid node_id is a programming error.
+
+        let node = self
+            .nodes
+            .get_mut(node_id)
+            .expect("Node to-be split does not exist");
+        let mut parent_key = node.key.split_off(prefix_len);
+        let mut parent_blocks = node.blocks.split_off(prefix_len);
+
+        // Move first part of the prefix to the parent. We swap to avoid
+        // an allocation + copy for both splits of the key/blocks.
+        std::mem::swap(&mut node.key, &mut parent_key);
+        std::mem::swap(&mut node.blocks, &mut parent_blocks);
+
+        let node_key = node.key[0];
+
+        let grandparent_id = node.parent.expect("Node does not have a parent");
+        let parent_id = self.add_node(grandparent_id, parent_key, parent_blocks);
+        self.add_node_to_parent(parent_id, node_key, node_id);
+
+        // Reborrow to make the borrow checker happy.
+        let node = self
+            .nodes
+            .get_mut(node_id)
+            .expect("Node to-be split does not exist");
+        node.parent = Some(parent_id);
+
+        parent_id
+    }
+
+    /// Create a node and add it to the parent.
+    fn add_node(
+        &mut self,
+        parent_id: NodeId,
+        key: impl Into<Vec<u32>>,
+        blocks: impl Into<Vec<u32>>,
+    ) -> NodeId {
+        let key = key.into();
+        let blocks = blocks.into();
+        let first = key[0];
+
+        let child = TrieNode::new(key, blocks, self.time, Some(parent_id));
+        let child_id = self.nodes.insert(child);
+
+        self.add_node_to_parent(parent_id, first, child_id);
+        self.leaves.insert((self.time, child_id));
+
+        child_id
+    }
+
+    /// Add a node to the parent.
+    fn add_node_to_parent(&mut self, parent_id: NodeId, first: u32, child_id: NodeId) {
+        // Unwrap here, passing in an unknown id is a programming error.
+        let parent = self.nodes.get_mut(parent_id).expect("Unknown parent node");
+        if parent.children.insert(first, child_id).is_none() {
+            // Only increase reference count if child does not replace another child.
+            self.incref(parent_id)
+                .expect("Failed to increase parent refcount");
+        }
+    }
+
+    /// Remove a node from the trie.
+    fn remove_node(&mut self, node_id: NodeId) -> TrieNode {
+        // Unwrap here, passing in an unknown id is a programming error.
+        let node = self.nodes.remove(node_id).expect("Unknown node");
+        let parent_id = node.parent.expect("Attempted to remove root node");
+        let parent = self.nodes.get_mut(parent_id).expect("Unknown parent node");
+        parent.children.remove(&node.key[0]);
+        self.decref(parent_id)
+            .expect("Failed to decrease parent refcount");
+        self.nodes.remove(node_id);
+        node
+    }
+
+    fn update_access_time(&mut self, node_id: NodeId) {
+        // Unwrap here, passing in an unknown id is a programming error.
+        let node = self.nodes.get_mut(node_id).expect("Unknown node");
+
+        // Update the ordered leaves set if the node is a leave.
+        if self.leaves.remove(&(node.last_accessed, node_id)) {
+            self.leaves.insert((self.time, node_id));
+        }
+
+        node.last_accessed = self.time;
+    }
+
+    #[allow(dead_code)]
+    #[doc(hidden)]
+    /// Print debugging output for the trie.
+    ///
+    /// In contrast to `Debug` nicely formatted.
+    pub fn print_debug(&self) {
+        self.print_debug_(self.root, 0);
+    }
+
+    fn print_debug_(&self, node_id: NodeId, indent: usize) {
+        let node = &self.nodes[node_id];
+        eprintln!(
+            "{}{:?}, key: {:?}, blocks: {:?}, ref_count: {}, last_accessed: {}, parent: {:?}, children: {:?}",
+            " ".repeat(indent),
+            node_id,
+            node.key,
+            node.blocks,
+            node.ref_count,
+            node.last_accessed,
+            node.parent,
+            node.children
+        );
+        for child_id in self.nodes[node_id].children.values() {
+            self.print_debug_(*child_id, indent + 2);
+        }
+    }
+
+    pub(crate) fn root_id(&self) -> DefaultKey {
+        self.root
+    }
+}
+
+/// Trie node.
+#[derive(Debug)]
+struct TrieNode {
+    blocks: Vec<u32>,
+    children: HashMap<u32, NodeId>,
+    key: Vec<u32>,
+    last_accessed: u64,
+    parent: Option<NodeId>,
+    ref_count: usize,
+}
+
+impl TrieNode {
+    fn new(key: Vec<u32>, blocks: Vec<u32>, last_accessed: u64, parent: Option<NodeId>) -> Self {
+        TrieNode {
+            children: HashMap::new(),
+            key,
+            blocks,
+            last_accessed,
+            parent,
+            ref_count: 0,
+        }
+    }
+}
+
+/// Helper trait to get the length of the shared prefix of two sequences.
+trait SharedPrefixLen {
+    fn shared_prefix_len(&self, other: &Self) -> usize;
+}
+
+impl<T> SharedPrefixLen for [T]
+where
+    T: PartialEq,
+{
+    fn shared_prefix_len(&self, other: &Self) -> usize {
+        self.iter().zip(other).take_while(|(a, b)| a == b).count()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use crate::block_allocator::Allocator;
+
+    use super::RadixAllocator;
+
+    #[test]
+    fn allocator_reuses_prefixes() {
+        let mut cache = RadixAllocator::new(1, 12, None);
+        let allocation = cache.allocate(8, Some(Arc::new(vec![0, 1, 2, 3]))).unwrap();
+        assert_eq!(allocation.blocks, vec![4, 5, 6, 7, 8, 9, 10, 11]);
+        assert_eq!(allocation.slots, allocation.slots);
+        assert_eq!(allocation.prefix_len, 0);
+        cache.free(allocation.blocks.clone(), allocation.allocation_id);
+
+        let allocation = cache.allocate(8, Some(Arc::new(vec![0, 1, 2, 3]))).unwrap();
+        assert_eq!(allocation.blocks, vec![4, 5, 6, 7, 8, 9, 10, 11]);
+        assert_eq!(allocation.prefix_len, 4);
+    }
+
+    #[test]
+    fn allocator_collects_older_prefixes_first() {
+        let mut cache = RadixAllocator::new(1, 7, None);
+        let allocation1 = cache.allocate(4, Some(Arc::new(vec![0, 1, 2, 3]))).unwrap();
+        assert_eq!(allocation1.blocks, vec![3, 4, 5, 6]);
+        assert_eq!(allocation1.prefix_len, 0);
+
+        let allocation2 = cache.allocate(2, Some(Arc::new(vec![4, 5]))).unwrap();
+        assert_eq!(allocation2.blocks, vec![1, 2]);
+        assert_eq!(allocation2.prefix_len, 0);
+
+        cache.free(allocation1.blocks.clone(), allocation1.allocation_id);
+        cache.free(allocation2.blocks.clone(), allocation2.allocation_id);
+
+        // We should get the blocks of the first allocation, since they are more recent.
+        let allocation3 = cache.allocate(4, Some(Arc::new(vec![6, 7, 8, 9]))).unwrap();
+        assert_eq!(allocation3.blocks, vec![3, 4, 5, 6]);
+        assert_eq!(allocation3.prefix_len, 0);
+    }
+
+    #[test]
+    fn allocator_frees_fully_overlapping_prefills() {
+        let mut cache = RadixAllocator::new(1, 10, None);
+        let allocation1 = cache.allocate(4, Some(Arc::new(vec![0, 1, 2, 3]))).unwrap();
+        let allocation2 = cache.allocate(4, Some(Arc::new(vec![0, 1, 2, 3]))).unwrap();
+
+        cache.free(allocation2.blocks.clone(), allocation2.allocation_id);
+        cache.free(allocation1.blocks.clone(), allocation1.allocation_id);
+
+        let allocation3 = cache.allocate(4, Some(Arc::new(vec![0, 1, 2, 3]))).unwrap();
+        assert_eq!(allocation3.prefix_len, 4);
+
+        // 10 blocks, of which 1 reserved for health checks, 4 for the cached blocks.
+        assert_eq!(cache.free_blocks.len(), 5);
+    }
+
+    #[test]
+    fn allocator_frees_partially_overlapping_prefills() {
+        let mut cache = RadixAllocator::new(1, 20, None);
+        let allocation1 = cache.allocate(4, Some(Arc::new(vec![0, 1]))).unwrap();
+        assert_eq!(allocation1.blocks, vec![16, 17, 18, 19]);
+        assert_eq!(allocation1.prefix_len, 0);
+
+        cache.free(allocation1.blocks.clone(), allocation1.allocation_id);
+
+        let allocation2 = cache
+            .allocate(8, Some(Arc::new(vec![0, 1, 2, 3, 4, 5])))
+            .unwrap();
+        assert_eq!(allocation2.blocks, vec![16, 17, 12, 13, 14, 15, 18, 19]);
+        assert_eq!(allocation2.prefix_len, 2);
+
+        let allocation3 = cache
+            .allocate(8, Some(Arc::new(vec![0, 1, 2, 3, 6, 7])))
+            .unwrap();
+        assert_eq!(allocation3.blocks, vec![16, 17, 6, 7, 8, 9, 10, 11]);
+        assert_eq!(allocation3.prefix_len, 2);
+
+        cache.free(allocation3.blocks.clone(), allocation3.allocation_id);
+        cache.free(allocation2.blocks.clone(), allocation2.allocation_id);
+
+        // 20 blocks, of which 1 reserved for health checks, 6 for allocation3, 2 for allocation2.
+        assert_eq!(cache.free_blocks.len(), 11);
+
+        let allocation4 = cache
+            .allocate(6, Some(Arc::new(vec![0, 1, 2, 3, 4, 5])))
+            .unwrap();
+        assert_eq!(allocation4.blocks, vec![16, 17, 6, 7, 14, 15]);
+        assert_eq!(allocation4.prefix_len, 6);
+        assert_eq!(cache.free_blocks.len(), 11);
+
+        let allocation5 = cache
+            .allocate(6, Some(Arc::new(vec![0, 1, 2, 3, 6, 7])))
+            .unwrap();
+        assert_eq!(allocation5.blocks, vec![16, 17, 6, 7, 8, 9]);
+        assert_eq!(allocation5.prefix_len, 6);
+        assert_eq!(cache.free_blocks.len(), 11);
+    }
+
+    #[test]
+    fn trie_insertions_have_correct_prefix_len() {
+        let mut trie = super::RadixTrie::new();
+
+        assert_eq!(trie.insert(&[0, 1, 2], &[0, 1, 2]).unwrap(), 0);
+
+        // Already exists.
+        assert_eq!(trie.insert(&[0, 1, 2], &[0, 1, 2]).unwrap(), 3);
+
+        // Completely new at root-level
+        assert_eq!(trie.insert(&[1, 2, 3], &[1, 2, 3]).unwrap(), 0);
+
+        // Contains full prefix, but longer.
+        assert_eq!(trie.insert(&[0, 1, 2, 3, 4], &[0, 1, 2, 3, 4]).unwrap(), 3);
+
+        // Shares partial prefix, we need a split.
+        assert_eq!(
+            trie.insert(&[0, 1, 2, 3, 5, 6, 7], &[0, 1, 2, 3, 5, 6, 7])
+                .unwrap(),
+            4
+        );
+    }
+
+    #[test]
+    fn trie_get_returns_correct_blocks() {
+        let mut trie = super::RadixTrie::new();
+        trie.insert(&[0, 1, 2], &[0, 1, 2]).unwrap();
+        trie.insert(&[1, 2, 3], &[1, 2, 3]).unwrap();
+        trie.insert(&[0, 1, 2, 3, 4], &[0, 1, 2, 3, 4]).unwrap();
+        trie.insert(&[0, 1, 2, 3, 5, 6, 7], &[0, 1, 2, 3, 5, 6, 7])
+            .unwrap();
+
+        let mut blocks = Vec::new();
+        trie.find(&[0], &mut blocks);
+        assert_eq!(blocks, vec![0]);
+
+        blocks.clear();
+        trie.find(&[0, 1, 2], &mut blocks);
+        assert_eq!(blocks, vec![0, 1, 2]);
+
+        blocks.clear();
+        trie.find(&[1, 2, 3], &mut blocks);
+        assert_eq!(blocks, vec![1, 2, 3]);
+
+        blocks.clear();
+        trie.find(&[0, 1, 2, 3], &mut blocks);
+        assert_eq!(blocks, vec![0, 1, 2, 3]);
+
+        blocks.clear();
+        trie.find(&[0, 1, 2, 3, 4], &mut blocks);
+        assert_eq!(blocks, vec![0, 1, 2, 3, 4]);
+
+        blocks.clear();
+        trie.find(&[0, 1, 2, 3, 5], &mut blocks);
+        assert_eq!(blocks, vec![0, 1, 2, 3, 5]);
+    }
+
+    #[test]
+    fn trie_evict_removes_correct_blocks() {
+        let mut trie = super::RadixTrie::new();
+        trie.insert(&[0, 1, 2], &[0, 1, 2]).unwrap();
+        trie.insert(&[0, 1, 2, 3, 5, 6, 7], &[0, 1, 2, 3, 5, 6, 7])
+            .unwrap();
+        trie.insert(&[0, 1, 2, 3, 4], &[0, 1, 2, 3, 4]).unwrap();
+        trie.insert(&[1, 2, 3], &[1, 2, 3]).unwrap();
+
+        let mut blocks = Vec::new();
+
+        // Remove less than the leave blocks.
+        assert_eq!(trie.evict(1), vec![7]);
+        trie.find(&[0, 1, 2, 3, 5, 6, 7], &mut blocks);
+        assert_eq!(blocks, vec![0, 1, 2, 3, 5, 6]);
+
+        // Refresh other leaf.
+        trie.find(&[0, 1, 2, 3, 4], &mut blocks);
+        trie.find(&[1, 2, 3], &mut blocks);
+
+        // Remove the leave blocks exactly.
+        assert_eq!(trie.evict(2), vec![5, 6]);
+        blocks.clear();
+        trie.find(&[0, 1, 2, 3, 5, 6, 7], &mut blocks);
+        assert_eq!(blocks, vec![0, 1, 2, 3]);
+
+        trie.find(&[1, 2, 3], &mut blocks);
+
+        // Remove more than the leave blocks.
+        assert_eq!(trie.evict(3), vec![4, 3, 2]);
+        blocks.clear();
+        trie.find(&[0, 1, 2, 3, 4], &mut blocks);
+        assert_eq!(blocks, vec![0, 1]);
+
+        // Clear out the whole trie.
+        assert_eq!(trie.evict(10), vec![1, 2, 3, 0, 1]);
+    }
+}
diff --git a/benchmark/src/generation.rs b/benchmark/src/generation.rs
index 5e739703..7494d5b5 100644
--- a/benchmark/src/generation.rs
+++ b/benchmark/src/generation.rs
@@ -157,6 +157,7 @@ async fn prefill(
             top_n_tokens: top_n_tokens.unwrap_or(0),
             blocks: vec![],
             slots: vec![],
+            prefix_len: 0,
             adapter_id: None,
         })
         .collect();
diff --git a/proto/v3/generate.proto b/proto/v3/generate.proto
index 926c878e..68eea7ac 100644
--- a/proto/v3/generate.proto
+++ b/proto/v3/generate.proto
@@ -3,22 +3,23 @@ syntax = "proto3";
 package generate.v3;
 
 service TextGenerationService {
-    /// Model Info
-    rpc Info (InfoRequest) returns (InfoResponse) {}
-    /// Service discovery
-    rpc ServiceDiscovery (ServiceDiscoveryRequest) returns (ServiceDiscoveryResponse) {}
-    /// Empties batch cache
-    rpc ClearCache (ClearCacheRequest) returns (ClearCacheResponse);
-    /// Remove requests from a cached batch
-    rpc FilterBatch (FilterBatchRequest) returns (FilterBatchResponse);
-    /// Warmup the model and compute max cache size
-    rpc Warmup (WarmupRequest) returns (WarmupResponse);
-    /// Prefill batch and decode first token
-    rpc Prefill (PrefillRequest) returns (PrefillResponse);
-    /// Decode token for a list of prefilled batches
-    rpc Decode (DecodeRequest) returns (DecodeResponse);
-    /// Health check
-    rpc Health (HealthRequest) returns (HealthResponse);
+  /// Model Info
+  rpc Info(InfoRequest) returns (InfoResponse) {}
+  /// Service discovery
+  rpc ServiceDiscovery(ServiceDiscoveryRequest)
+      returns (ServiceDiscoveryResponse) {}
+  /// Empties batch cache
+  rpc ClearCache(ClearCacheRequest) returns (ClearCacheResponse);
+  /// Remove requests from a cached batch
+  rpc FilterBatch(FilterBatchRequest) returns (FilterBatchResponse);
+  /// Warmup the model and compute max cache size
+  rpc Warmup(WarmupRequest) returns (WarmupResponse);
+  /// Prefill batch and decode first token
+  rpc Prefill(PrefillRequest) returns (PrefillResponse);
+  /// Decode token for a list of prefilled batches
+  rpc Decode(DecodeRequest) returns (DecodeResponse);
+  /// Health check
+  rpc Health(HealthRequest) returns (HealthResponse);
 }
 
 message HealthRequest {}
@@ -28,240 +29,239 @@ message HealthResponse {}
 message InfoRequest {}
 
 message InfoResponse {
-    bool requires_padding = 1;
-    string dtype = 2;
-    string device_type = 3;
-    optional uint32 window_size = 4;
-    uint32 speculate = 5;
+  bool requires_padding = 1;
+  string dtype = 2;
+  string device_type = 3;
+  optional uint32 window_size = 4;
+  uint32 speculate = 5;
 }
 
 /// Empty request
 message ServiceDiscoveryRequest {}
 
 message ServiceDiscoveryResponse {
-    /// Other shards urls
-    repeated string urls = 1;
+  /// Other shards urls
+  repeated string urls = 1;
 }
 
 message ClearCacheRequest {
-    /// Optional batch id
-    optional uint64 id = 1;
+  /// Optional batch id
+  optional uint64 id = 1;
 }
 
 /// Empty response
 message ClearCacheResponse {}
 
 message Image {
-    /// Binary image data.
-    bytes data = 1;
+  /// Binary image data.
+  bytes data = 1;
 
-    /// Image MIME type.
-    string mimetype = 2;
+  /// Image MIME type.
+  string mimetype = 2;
 }
 
 message InputChunk {
-    oneof chunk {
-        /// Plain text data
-        string text = 1;
-        /// Image data
-        Image image = 2;
-    }
+  oneof chunk {
+    /// Plain text data
+    string text = 1;
+    /// Image data
+    Image image = 2;
+  }
 }
 
-message Input {
-    repeated InputChunk chunks = 1;
-  }
+message Input { repeated InputChunk chunks = 1; }
 
 enum GrammarType {
-    GRAMMAR_TYPE_NONE = 0;
-    GRAMMAR_TYPE_JSON = 1;
-    GRAMMAR_TYPE_REGEX = 2;
+  GRAMMAR_TYPE_NONE = 0;
+  GRAMMAR_TYPE_JSON = 1;
+  GRAMMAR_TYPE_REGEX = 2;
 }
 
 message NextTokenChooserParameters {
-    /// exponential scaling output probability distribution
-    float temperature = 1;
-    /// restricting to the k highest probability elements
-    uint32 top_k = 2;
-    /// restricting to top tokens summing to prob_cut_off <= prob_cut_off
-    float top_p = 3;
-    /// restricting to top tokens summing to prob_cut_off <= prob_cut_off
-    float typical_p = 4;
-    /// apply sampling on the logits
-    bool do_sample = 5;
-    /// random seed for sampling
-    uint64 seed = 6;
-    /// repetition penalty
-    float repetition_penalty = 7;
-    /// frequency penalty
-    float frequency_penalty = 9;
-    /// token watermarking using "A Watermark for Large Language Models"
-    bool watermark = 8;
-    /// grammar (applied if not empty)
-    string grammar = 10;
-    /// grammar type
-    GrammarType grammar_type = 11;
+  /// exponential scaling output probability distribution
+  float temperature = 1;
+  /// restricting to the k highest probability elements
+  uint32 top_k = 2;
+  /// restricting to top tokens summing to prob_cut_off <= prob_cut_off
+  float top_p = 3;
+  /// restricting to top tokens summing to prob_cut_off <= prob_cut_off
+  float typical_p = 4;
+  /// apply sampling on the logits
+  bool do_sample = 5;
+  /// random seed for sampling
+  uint64 seed = 6;
+  /// repetition penalty
+  float repetition_penalty = 7;
+  /// frequency penalty
+  float frequency_penalty = 9;
+  /// token watermarking using "A Watermark for Large Language Models"
+  bool watermark = 8;
+  /// grammar (applied if not empty)
+  string grammar = 10;
+  /// grammar type
+  GrammarType grammar_type = 11;
 }
 
 message StoppingCriteriaParameters {
-    /// Maximum number of generated tokens
-    uint32 max_new_tokens = 1;
-    /// Optional stopping sequences
-    repeated string stop_sequences = 2;
-    /// Ignore end of sequence token
-    /// used for benchmarking
-    bool ignore_eos_token = 3;
+  /// Maximum number of generated tokens
+  uint32 max_new_tokens = 1;
+  /// Optional stopping sequences
+  repeated string stop_sequences = 2;
+  /// Ignore end of sequence token
+  /// used for benchmarking
+  bool ignore_eos_token = 3;
 }
 
 message Request {
-    /// Request ID
-    uint64 id = 1;
-    /// The generation context as chunks
-    Input input_chunks = 8;
-    /// The generation context, stringified input_chunks
-    string inputs = 2;
-    /// Context truncation
-    uint32 truncate = 3;
-    /// Next Token Chooser Parameters
-    NextTokenChooserParameters parameters = 4;
-    /// Stopping Criteria Parameters
-    StoppingCriteriaParameters stopping_parameters = 5;
-    /// Return prefill logprobs
-    bool prefill_logprobs = 6;
-    /// Return most likely n tokens
-    uint32 top_n_tokens = 7;
-    /// Paged attention blocks
-    repeated uint32 blocks = 9;
-    /// Paged attention slots
-    repeated uint32  slots = 10;
-    /// LORA adapter index
-    optional string adapter_id = 11;
+  /// Request ID
+  uint64 id = 1;
+  /// The generation context as chunks
+  Input input_chunks = 8;
+  /// The generation context, stringified input_chunks
+  string inputs = 2;
+  /// Context truncation
+  uint32 truncate = 3;
+  /// Next Token Chooser Parameters
+  NextTokenChooserParameters parameters = 4;
+  /// Stopping Criteria Parameters
+  StoppingCriteriaParameters stopping_parameters = 5;
+  /// Return prefill logprobs
+  bool prefill_logprobs = 6;
+  /// Return most likely n tokens
+  uint32 top_n_tokens = 7;
+  /// Paged attention blocks
+  repeated uint32 blocks = 9;
+  /// Paged attention slots
+  repeated uint32 slots = 10;
+  /// LORA adapter index
+  optional string adapter_id = 11;
+  /// Prefix length that can be retrieved from the KV cache.
+  uint32 prefix_len = 12;
 }
 
 message Batch {
-    /// Batch ID
-    uint64 id = 1;
-    /// Individual requests
-    repeated Request requests = 2;
-    /// Batch size (==len(requests))
-    uint32 size = 3;
-    /// Maximum number of tokens this batch will grow to
-    uint32 max_tokens = 4;
-    /// Maximum number of Paged Attention blocks
-    uint32 max_blocks = 5;
+  /// Batch ID
+  uint64 id = 1;
+  /// Individual requests
+  repeated Request requests = 2;
+  /// Batch size (==len(requests))
+  uint32 size = 3;
+  /// Maximum number of tokens this batch will grow to
+  uint32 max_tokens = 4;
+  /// Maximum number of Paged Attention blocks
+  uint32 max_blocks = 5;
 }
 
 message CachedBatch {
-    /// Batch ID
-    uint64 id = 1;
-    /// Individual requests ids
-    repeated uint64 request_ids = 2;
-    /// Batch size (==len(requests))
-    uint32 size = 3;
-    /// Maximum number of tokens this batch will grow to
-    uint32 max_tokens = 4;
+  /// Batch ID
+  uint64 id = 1;
+  /// Individual requests ids
+  repeated uint64 request_ids = 2;
+  /// Batch size (==len(requests))
+  uint32 size = 3;
+  /// Maximum number of tokens this batch will grow to
+  uint32 max_tokens = 4;
 }
 
 enum FinishReason {
-    FINISH_REASON_LENGTH = 0;
-    FINISH_REASON_EOS_TOKEN = 1;
-    FINISH_REASON_STOP_SEQUENCE = 2;
+  FINISH_REASON_LENGTH = 0;
+  FINISH_REASON_EOS_TOKEN = 1;
+  FINISH_REASON_STOP_SEQUENCE = 2;
 }
 
 message GeneratedText {
-    /// Output
-    string text = 1;
-    /// Number of generated tokens
-    uint32 generated_tokens = 2;
-    /// Finish reason
-    FinishReason finish_reason = 3;
-    /// Seed
-    optional uint64 seed = 4;
+  /// Output
+  string text = 1;
+  /// Number of generated tokens
+  uint32 generated_tokens = 2;
+  /// Finish reason
+  FinishReason finish_reason = 3;
+  /// Seed
+  optional uint64 seed = 4;
 }
 
 message Tokens {
-    /// Token IDs
-    repeated uint32 ids = 1;
-    /// Logprobs
-    repeated float logprobs = 2;
-    /// tokens
-    repeated string texts = 3;
-    /// special
-    repeated bool is_special = 4;
+  /// Token IDs
+  repeated uint32 ids = 1;
+  /// Logprobs
+  repeated float logprobs = 2;
+  /// tokens
+  repeated string texts = 3;
+  /// special
+  repeated bool is_special = 4;
 }
 
 message Generation {
-    /// Request ID
-    uint64 request_id = 1;
-    /// Prefill tokens (optional)
-    Tokens prefill_tokens = 2;
-    Tokens tokens = 3;
-    /// Complete generated text
-    optional GeneratedText generated_text = 4;
-    /// Top tokens
-    repeated Tokens top_tokens = 5;
+  /// Request ID
+  uint64 request_id = 1;
+  /// Prefill tokens (optional)
+  Tokens prefill_tokens = 2;
+  Tokens tokens = 3;
+  /// Complete generated text
+  optional GeneratedText generated_text = 4;
+  /// Top tokens
+  repeated Tokens top_tokens = 5;
 }
 
 message FilterBatchRequest {
-    /// Batch ID
-    uint64 batch_id = 1;
-    /// Requests to keep
-    repeated uint64 request_ids = 2;
+  /// Batch ID
+  uint64 batch_id = 1;
+  /// Requests to keep
+  repeated uint64 request_ids = 2;
 }
 
 message FilterBatchResponse {
-    /// Filtered Batch (cached)
-    CachedBatch batch = 1;
+  /// Filtered Batch (cached)
+  CachedBatch batch = 1;
 }
 
-
 message PrefillRequest {
-    /// Batch
-    Batch batch = 1;
+  /// Batch
+  Batch batch = 1;
 }
 
 message PrefillResponse {
-    /// Generation
-    repeated Generation generations = 1;
-    /// Next batch (cached)
-    optional CachedBatch batch = 2;
-    /// Forward elapsed time in nanoseconds
-    uint64 forward_ns = 3;
-    /// Decode elapsed time in nanoseconds
-    uint64 decode_ns = 4;
-    /// Total elapsed time in nanoseconds
-    uint64 total_ns = 5;
+  /// Generation
+  repeated Generation generations = 1;
+  /// Next batch (cached)
+  optional CachedBatch batch = 2;
+  /// Forward elapsed time in nanoseconds
+  uint64 forward_ns = 3;
+  /// Decode elapsed time in nanoseconds
+  uint64 decode_ns = 4;
+  /// Total elapsed time in nanoseconds
+  uint64 total_ns = 5;
 }
 
 message DecodeRequest {
-    /// Cached batches
-    repeated CachedBatch batches = 1;
+  /// Cached batches
+  repeated CachedBatch batches = 1;
 }
 
 message DecodeResponse {
-    /// Decodes
-    repeated Generation generations = 1;
-    /// Next batch (cached)
-    optional CachedBatch batch = 2;
-    /// Forward elapsed time in nanoseconds
-    uint64 forward_ns = 3;
-    /// Decode elapsed time in nanoseconds
-    uint64 decode_ns = 4;
-    /// Total elapsed time in nanoseconds
-    uint64 total_ns = 5;
-    /// Concatenate elapsed time in nanoseconds
-    optional uint64 concat_ns = 6;
+  /// Decodes
+  repeated Generation generations = 1;
+  /// Next batch (cached)
+  optional CachedBatch batch = 2;
+  /// Forward elapsed time in nanoseconds
+  uint64 forward_ns = 3;
+  /// Decode elapsed time in nanoseconds
+  uint64 decode_ns = 4;
+  /// Total elapsed time in nanoseconds
+  uint64 total_ns = 5;
+  /// Concatenate elapsed time in nanoseconds
+  optional uint64 concat_ns = 6;
 }
 
 message WarmupRequest {
-    /// Batch to warmup on
-    Batch batch = 1;
-    uint32 max_input_length = 2;
-    uint32 max_prefill_tokens = 3;
-    uint32 max_total_tokens = 4;
+  /// Batch to warmup on
+  Batch batch = 1;
+  uint32 max_input_length = 2;
+  uint32 max_prefill_tokens = 3;
+  uint32 max_total_tokens = 4;
 }
 
 message WarmupResponse {
-    /// Maximum number of tokens supported by the model
-    optional uint32 max_supported_total_tokens = 1;
+  /// Maximum number of tokens supported by the model
+  optional uint32 max_supported_total_tokens = 1;
 }
diff --git a/router/src/validation.rs b/router/src/validation.rs
index 3d1a4103..5011158a 100644
--- a/router/src/validation.rs
+++ b/router/src/validation.rs
@@ -11,6 +11,7 @@ use rand::{thread_rng, Rng};
 use serde_json::Value;
 use std::io::Cursor;
 use std::iter;
+use std::sync::Arc;
 use thiserror::Error;
 use tokenizers::tokenizer::Tokenizer;
 use tokio::sync::mpsc;
@@ -115,13 +116,14 @@ impl Validation {
         }
     }
 
+    #[allow(clippy::type_complexity)]
     #[instrument(skip(self, inputs))]
     async fn validate_input(
         &self,
         inputs: String,
         truncate: Option<usize>,
         max_new_tokens: Option<u32>,
-    ) -> Result<(Vec<Chunk>, usize, u32), ValidationError> {
+    ) -> Result<(Vec<Chunk>, Option<Vec<u32>>, usize, u32), ValidationError> {
         // If we have a fast tokenizer
         if let Some((encoding, inputs)) = self.tokenize(inputs.clone(), truncate).await? {
             // Create response channel
@@ -156,8 +158,10 @@ impl Validation {
                 ));
             }
 
+            let input_ids = encoding.get_ids()[..input_length].to_owned();
+
             metrics::histogram!("tgi_request_input_length").record(input_length as f64);
-            Ok((inputs, input_length, max_new_tokens))
+            Ok((inputs, Some(input_ids), input_length, max_new_tokens))
         }
         // Return inputs without validation
         else {
@@ -180,7 +184,12 @@ impl Validation {
                 input_length = input_length.saturating_sub(max_new_tokens as usize);
             }
 
-            Ok((vec![Chunk::Text(inputs)], input_length, max_new_tokens))
+            Ok((
+                vec![Chunk::Text(inputs)],
+                None,
+                input_length,
+                max_new_tokens,
+            ))
         }
     }
 
@@ -314,7 +323,7 @@ impl Validation {
             .unwrap_or(Ok(None))?;
 
         // Validate inputs
-        let (inputs, input_length, max_new_tokens) = self
+        let (inputs, input_ids, input_length, max_new_tokens) = self
             .validate_input(request.inputs, truncate, max_new_tokens)
             .await?;
 
@@ -391,6 +400,7 @@ impl Validation {
 
         Ok(ValidGenerateRequest {
             inputs,
+            input_ids: input_ids.map(Arc::new),
             decoder_input_details,
             input_length: input_length as u32,
             truncate: truncate.unwrap_or(self.max_input_length) as u32,
@@ -707,6 +717,7 @@ pub struct ValidStoppingParameters {
 #[derive(Debug, Clone)]
 pub struct ValidGenerateRequest {
     pub inputs: Vec<Chunk>,
+    pub input_ids: Option<Arc<Vec<u32>>>,
     pub input_length: u32,
     pub truncate: u32,
     pub decoder_input_details: bool,
diff --git a/server/text_generation_server/models/globals.py b/server/text_generation_server/models/globals.py
index b58a5b80..abc35421 100644
--- a/server/text_generation_server/models/globals.py
+++ b/server/text_generation_server/models/globals.py
@@ -5,16 +5,29 @@ from typing import Dict, Optional
 
 from text_generation_server.utils.log import log_master
 
-ATTENTION = os.getenv("ATTENTION", "paged")
+PREFIX_CACHING = os.getenv("USE_PREFIX_CACHING", False)
+log_master(logger.info, f"Using Attention = {PREFIX_CACHING}")
+
+ATTENTION = os.getenv("ATTENTION", "flashinfer" if PREFIX_CACHING else "paged")
 _expected = {"paged", "flashdecoding", "flashinfer"}
 assert (
     ATTENTION in _expected
 ), f"Attention is not valid {ATTENTION}, expected {_expected}"
 log_master(logger.info, f"Using Attention = {ATTENTION}")
 
+if PREFIX_CACHING and ATTENTION != "flashinfer":
+    raise RuntimeError("Prefix caching is only supported with flashinfer")
+
 MEM_POOL = torch.cuda.graph_pool_handle() if torch.cuda.is_available() else None
+
 # This is overridden by the cli
-BLOCK_SIZE: int = 256 if ATTENTION == "flashdecoding" else 16
+BLOCK_SIZE: int
+if ATTENTION == "flashdecoding":
+    BLOCK_SIZE = 256
+elif ATTENTION == "flashinfer":
+    BLOCK_SIZE = 1
+else:
+    BLOCK_SIZE = 16
 
 
 cuda_graphs = os.getenv("CUDA_GRAPHS")

From 136bcc812870e36aea69c3bb9cb8012f0a63d973 Mon Sep 17 00:00:00 2001
From: Nicolas Patry <patry.nicolas@protonmail.com>
Date: Mon, 12 Aug 2024 15:22:02 +0200
Subject: [PATCH 33/72] Keeping the benchmark somewhere (#2401)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Daniël de Kok <me@danieldk.eu>
---
 Cargo.lock                          | 171 ++++++++++++++++++++++++++--
 backends/v3/Cargo.toml              |  22 +++-
 backends/v3/benches/prefix_cache.rs |  47 ++++++++
 backends/v3/src/block_allocator.rs  |   6 +-
 backends/v3/src/lib.rs              |   4 +-
 backends/v3/src/queue.rs            |   2 +-
 backends/v3/src/radix.rs            |   5 +
 router/Cargo.toml                   |  20 +++-
 8 files changed, 255 insertions(+), 22 deletions(-)
 create mode 100644 backends/v3/benches/prefix_cache.rs

diff --git a/Cargo.lock b/Cargo.lock
index 3a5845a7..bb63422e 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -180,6 +180,17 @@ version = "1.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"
 
+[[package]]
+name = "atty"
+version = "0.2.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
+dependencies = [
+ "hermit-abi 0.1.19",
+ "libc",
+ "winapi",
+]
+
 [[package]]
 name = "autocfg"
 version = "1.3.0"
@@ -565,6 +576,12 @@ version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "df8670b8c7b9dae1793364eafadf7239c40d669904660c5960d74cfd80b46a53"
 
+[[package]]
+name = "cast"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
+
 [[package]]
 name = "cc"
 version = "1.1.7"
@@ -617,6 +634,17 @@ dependencies = [
  "libloading",
 ]
 
+[[package]]
+name = "clap"
+version = "2.34.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c"
+dependencies = [
+ "bitflags 1.3.2",
+ "textwrap",
+ "unicode-width",
+]
+
 [[package]]
 name = "clap"
 version = "4.5.11"
@@ -735,6 +763,42 @@ dependencies = [
  "cfg-if",
 ]
 
+[[package]]
+name = "criterion"
+version = "0.3.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b01d6de93b2b6c65e17c634a26653a29d107b3c98c607c765bf38d041531cd8f"
+dependencies = [
+ "atty",
+ "cast",
+ "clap 2.34.0",
+ "criterion-plot",
+ "csv",
+ "itertools 0.10.5",
+ "lazy_static",
+ "num-traits",
+ "oorandom",
+ "plotters",
+ "rayon",
+ "regex",
+ "serde",
+ "serde_cbor",
+ "serde_derive",
+ "serde_json",
+ "tinytemplate",
+ "walkdir",
+]
+
+[[package]]
+name = "criterion-plot"
+version = "0.4.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2673cc8207403546f45f5fd319a974b1e6983ad1a3ee7e6041650013be041876"
+dependencies = [
+ "cast",
+ "itertools 0.10.5",
+]
+
 [[package]]
 name = "crossbeam-channel"
 version = "0.5.13"
@@ -1060,7 +1124,7 @@ checksum = "887d93f60543e9a9362ef8a21beedd0a833c5d9610e18c67abe15a5963dcb1a4"
 dependencies = [
  "bit_field",
  "flume",
- "half",
+ "half 2.4.1",
  "lebe",
  "miniz_oxide",
  "rayon-core",
@@ -1367,6 +1431,12 @@ dependencies = [
  "tracing",
 ]
 
+[[package]]
+name = "half"
+version = "1.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1b43ede17f21864e81be2fa654110bf1e793774238d86ef8555c37e6519c0403"
+
 [[package]]
 name = "half"
 version = "2.4.1"
@@ -1404,6 +1474,15 @@ version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
 
+[[package]]
+name = "hermit-abi"
+version = "0.1.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "hermit-abi"
 version = "0.3.9"
@@ -1804,6 +1883,15 @@ dependencies = [
  "either",
 ]
 
+[[package]]
+name = "itertools"
+version = "0.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186"
+dependencies = [
+ "either",
+]
+
 [[package]]
 name = "itoa"
 version = "1.0.11"
@@ -1844,7 +1932,7 @@ dependencies = [
  "anyhow",
  "base64 0.21.7",
  "bytecount",
- "clap",
+ "clap 4.5.11",
  "fancy-regex",
  "fraction",
  "getrandom",
@@ -2132,7 +2220,7 @@ version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4569e456d394deccd22ce1c1913e6ea0e54519f577285001215d33557431afe4"
 dependencies = [
- "hermit-abi",
+ "hermit-abi 0.3.9",
  "libc",
  "wasi",
  "windows-sys 0.52.0",
@@ -2400,7 +2488,7 @@ version = "1.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43"
 dependencies = [
- "hermit-abi",
+ "hermit-abi 0.3.9",
  "libc",
 ]
 
@@ -2456,6 +2544,12 @@ dependencies = [
  "pkg-config",
 ]
 
+[[package]]
+name = "oorandom"
+version = "11.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9"
+
 [[package]]
 name = "openssl"
 version = "0.10.66"
@@ -2783,6 +2877,34 @@ version = "0.3.30"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec"
 
+[[package]]
+name = "plotters"
+version = "0.3.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a15b6eccb8484002195a3e44fe65a4ce8e93a625797a063735536fd59cb01cf3"
+dependencies = [
+ "num-traits",
+ "plotters-backend",
+ "plotters-svg",
+ "wasm-bindgen",
+ "web-sys",
+]
+
+[[package]]
+name = "plotters-backend"
+version = "0.3.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "414cec62c6634ae900ea1c56128dfe87cf63e7caece0852ec76aba307cebadb7"
+
+[[package]]
+name = "plotters-svg"
+version = "0.3.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "81b30686a7d9c3e010b84284bdd26a29f2138574f52f5eb6f794fc0ad924e705"
+dependencies = [
+ "plotters-backend",
+]
+
 [[package]]
 name = "png"
 version = "0.17.13"
@@ -3525,6 +3647,16 @@ dependencies = [
  "serde_derive",
 ]
 
+[[package]]
+name = "serde_cbor"
+version = "0.11.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2bef2ebfde456fb76bbcf9f59315333decc4fda0b2b44b420243c11e0f5ec1f5"
+dependencies = [
+ "half 1.8.3",
+ "serde",
+]
+
 [[package]]
 name = "serde_derive"
 version = "1.0.204"
@@ -3891,7 +4023,7 @@ version = "2.2.1-dev0"
 dependencies = [
  "async-stream",
  "async-trait",
- "clap",
+ "clap 4.5.11",
  "cmake",
  "cxx",
  "cxx-build",
@@ -3912,7 +4044,7 @@ name = "text-generation-benchmark"
 version = "2.2.1-dev0"
 dependencies = [
  "average",
- "clap",
+ "clap 4.5.11",
  "crossterm",
  "float-ord",
  "hf-hub",
@@ -3950,7 +4082,7 @@ dependencies = [
 name = "text-generation-launcher"
 version = "2.2.1-dev0"
 dependencies = [
- "clap",
+ "clap 4.5.11",
  "ctrlc",
  "float_eq",
  "hf-hub",
@@ -3974,7 +4106,7 @@ dependencies = [
  "axum 0.7.5",
  "axum-tracing-opentelemetry",
  "base64 0.22.1",
- "clap",
+ "clap 4.5.11",
  "csv",
  "futures",
  "futures-util",
@@ -4022,13 +4154,15 @@ dependencies = [
  "axum 0.7.5",
  "axum-tracing-opentelemetry",
  "base64 0.22.1",
- "clap",
+ "clap 4.5.11",
+ "criterion",
  "futures",
  "futures-util",
  "grpc-metadata",
  "hf-hub",
  "image",
  "init-tracing-opentelemetry",
+ "itertools 0.13.0",
  "jsonschema",
  "metrics",
  "metrics-exporter-prometheus",
@@ -4062,6 +4196,15 @@ dependencies = [
  "utoipa-swagger-ui",
 ]
 
+[[package]]
+name = "textwrap"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060"
+dependencies = [
+ "unicode-width",
+]
+
 [[package]]
 name = "thiserror"
 version = "1.0.63"
@@ -4136,6 +4279,16 @@ dependencies = [
  "time-core",
 ]
 
+[[package]]
+name = "tinytemplate"
+version = "1.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc"
+dependencies = [
+ "serde",
+ "serde_json",
+]
+
 [[package]]
 name = "tinyvec"
 version = "1.8.0"
diff --git a/backends/v3/Cargo.toml b/backends/v3/Cargo.toml
index 129ceb9c..06a44bec 100644
--- a/backends/v3/Cargo.toml
+++ b/backends/v3/Cargo.toml
@@ -35,8 +35,14 @@ serde = "1.0.188"
 serde_json = "1.0.107"
 slotmap = "1.0.7"
 thiserror = "1.0.48"
-tokenizers = { workspace = true}
-tokio = { version = "1.32.0", features = ["rt", "rt-multi-thread", "parking_lot", "signal", "sync"] }
+tokenizers = { workspace = true }
+tokio = { version = "1.32.0", features = [
+  "rt",
+  "rt-multi-thread",
+  "parking_lot",
+  "signal",
+  "sync",
+] }
 tokio-stream = "0.1.14"
 tower-http = { version = "0.5.1", features = ["cors"] }
 tracing = "0.1.37"
@@ -44,7 +50,9 @@ tracing-opentelemetry = "0.21.0"
 tracing-subscriber = { version = "0.3.17", features = ["json", "env-filter"] }
 utoipa = { version = "4.2.0", features = ["axum_extras"] }
 utoipa-swagger-ui = { version = "6.0.0", features = ["axum"] }
-init-tracing-opentelemetry = { version = "0.14.1", features = ["opentelemetry-otlp"] }
+init-tracing-opentelemetry = { version = "0.14.1", features = [
+  "opentelemetry-otlp",
+] }
 minijinja = { version = "2.0.2" }
 minijinja-contrib = { version = "2.0.2", features = ["pycompat"] }
 futures-util = "0.3.30"
@@ -60,8 +68,16 @@ tower = "^0.4"
 tonic-build = "0.10.1"
 prost-build = "0.12.1"
 
+[dev-dependencies]
+criterion = "0.3"
+itertools = "0.13"
+
 [features]
 default = ["ngrok"]
 ngrok = ["text-generation-router/ngrok"]
 google = ["text-generation-router/google"]
 kserve = ["text-generation-router/kserve"]
+
+[[bench]]
+name = "prefix_cache"
+harness = false
diff --git a/backends/v3/benches/prefix_cache.rs b/backends/v3/benches/prefix_cache.rs
new file mode 100644
index 00000000..d9df45b2
--- /dev/null
+++ b/backends/v3/benches/prefix_cache.rs
@@ -0,0 +1,47 @@
+use std::sync::Arc;
+
+use criterion::{black_box, criterion_group, criterion_main, Criterion};
+use rand::Rng;
+
+use text_generation_router_v3::block_allocator::Allocator;
+use text_generation_router_v3::radix::RadixAllocator;
+
+fn prefix_cache_benchmark(c: &mut Criterion) {
+    // let prefixes: Vec<Vec<u32>> = (0..8192)
+    //     .chunks(256)
+    //     .into_iter()
+    //     .map(|c| c.collect())
+    //     .collect();
+
+    let mut cache = RadixAllocator::new(1, 262144, None);
+
+    c.bench_function("Radix allocator", |b| {
+        b.iter_batched(
+            || {
+                //prefixes
+                //    .choose_multiple(&mut rand::thread_rng(), 5)
+                //    .fold(Vec::new(), |mut v, s| {
+                //        v.extend(s);
+                //        v
+                //    })
+
+                (0..7936)
+                    .map(|_| rand::thread_rng().gen_range(0..1024))
+                    .collect::<Vec<u32>>()
+            },
+            |prefill| {
+                let alloc = cache.allocate(
+                    prefill.len() as u32 + 13,
+                    Some(Arc::new(black_box(prefill))),
+                );
+                if let Some(alloc) = alloc {
+                    cache.free(alloc.blocks.clone(), alloc.allocation_id);
+                }
+            },
+            criterion::BatchSize::SmallInput,
+        );
+    });
+}
+
+criterion_group!(benches, prefix_cache_benchmark);
+criterion_main!(benches);
diff --git a/backends/v3/src/block_allocator.rs b/backends/v3/src/block_allocator.rs
index 05c2bd30..c5503b8c 100644
--- a/backends/v3/src/block_allocator.rs
+++ b/backends/v3/src/block_allocator.rs
@@ -4,7 +4,7 @@ use tokio::sync::{mpsc, oneshot};
 use crate::radix::RadixAllocator;
 
 #[derive(Debug, Clone)]
-pub(crate) struct BlockAllocation {
+pub struct BlockAllocation {
     pub allocation_id: u64,
     pub blocks: Vec<u32>,
     pub slots: Vec<u32>,
@@ -25,7 +25,7 @@ impl Drop for BlockAllocation {
 }
 
 #[derive(Debug, Clone)]
-pub(crate) struct BlockAllocator {
+pub struct BlockAllocator {
     /// Channel to communicate with the background task
     block_allocator: mpsc::UnboundedSender<BlockAllocatorCommand>,
 }
@@ -128,7 +128,7 @@ enum BlockAllocatorCommand {
     },
 }
 
-pub(crate) trait Allocator {
+pub trait Allocator {
     fn allocate(
         &mut self,
         tokens: u32,
diff --git a/backends/v3/src/lib.rs b/backends/v3/src/lib.rs
index c8fc55f8..77a9a11a 100644
--- a/backends/v3/src/lib.rs
+++ b/backends/v3/src/lib.rs
@@ -1,8 +1,8 @@
 mod backend;
-mod block_allocator;
+pub mod block_allocator;
 mod client;
 mod queue;
-mod radix;
+pub mod radix;
 
 use crate::client::{ClientError, ShardedClient};
 pub(crate) use backend::BackendV3;
diff --git a/backends/v3/src/queue.rs b/backends/v3/src/queue.rs
index 13544235..0fb05a98 100644
--- a/backends/v3/src/queue.rs
+++ b/backends/v3/src/queue.rs
@@ -250,7 +250,7 @@ impl State {
 
         // Create span for this batch to add context to inference calls
         let next_batch_span = info_span!(parent: None, "batch", batch_size = tracing::field::Empty);
-        next_batch_span.follows_from(&Span::current());
+        next_batch_span.follows_from(Span::current());
 
         let mut batch_requests = Vec::with_capacity(self.entries.len());
         let mut batch_entries =
diff --git a/backends/v3/src/radix.rs b/backends/v3/src/radix.rs
index 0464b9f8..ef963532 100644
--- a/backends/v3/src/radix.rs
+++ b/backends/v3/src/radix.rs
@@ -205,6 +205,11 @@ pub struct RadixTrie {
     /// call that a real time lookup would require.
     time: u64,
 }
+impl Default for RadixTrie {
+    fn default() -> Self {
+        Self::new()
+    }
+}
 
 impl RadixTrie {
     /// Construct a new radix trie.
diff --git a/router/Cargo.toml b/router/Cargo.toml
index 1be74546..7773e212 100644
--- a/router/Cargo.toml
+++ b/router/Cargo.toml
@@ -27,8 +27,14 @@ reqwest = { version = "0.11.20", features = [] }
 serde = "1.0.188"
 serde_json = "1.0.107"
 thiserror = "1.0.48"
-tokenizers = { workspace = true}
-tokio = { version = "1.32.0", features = ["rt", "rt-multi-thread", "parking_lot", "signal", "sync"] }
+tokenizers = { workspace = true }
+tokio = { version = "1.32.0", features = [
+  "rt",
+  "rt-multi-thread",
+  "parking_lot",
+  "signal",
+  "sync",
+] }
 tokio-stream = "0.1.14"
 tower-http = { version = "0.5.1", features = ["cors"] }
 tracing = "0.1.40"
@@ -37,7 +43,9 @@ tracing-subscriber = { version = "0.3.18", features = ["json", "env-filter"] }
 utoipa = { version = "4.2.0", features = ["axum_extras"] }
 utoipa-swagger-ui = { version = "6.0.0", features = ["axum"] }
 ngrok = { version = "0.13.1", features = ["axum"], optional = true }
-init-tracing-opentelemetry = { version = "0.14.1", features = ["opentelemetry-otlp"] }
+init-tracing-opentelemetry = { version = "0.14.1", features = [
+  "opentelemetry-otlp",
+] }
 minijinja = { version = "2.0.2" }
 minijinja-contrib = { version = "2.0.2", features = ["pycompat"] }
 futures-util = "0.3.30"
@@ -46,7 +54,11 @@ once_cell = "1.19.0"
 image = "0.25.1"
 base64 = { workspace = true }
 sysinfo = "0.30.13"
-uuid = { version = "1.9.1", default-features = false, features = ["v4", "fast-rng", "macro-diagnostics"] }
+uuid = { version = "1.9.1", default-features = false, features = [
+  "v4",
+  "fast-rng",
+  "macro-diagnostics",
+] }
 csv = "1.3.0"
 ureq = "=2.9"
 

From 155f9c98e2a272d255d42f0da8d2e88cc1eaa18a Mon Sep 17 00:00:00 2001
From: drbh <david.richard.holtz@gmail.com>
Date: Mon, 12 Aug 2024 10:58:40 -0400
Subject: [PATCH 34/72] =?UTF-8?q?feat:=20validate=20template=20variables?=
 =?UTF-8?q?=20before=20apply=20and=20improve=20sliding=20wi=E2=80=A6=20(#2?=
 =?UTF-8?q?403)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* feat: validate template variables before apply and improve sliding window check

* fix: improve missing template var test
---
 router/src/infer/chat_template.rs             | 61 +++++++++++++++++--
 router/src/infer/mod.rs                       |  3 +
 router/src/server.rs                          |  1 +
 router/src/validation.rs                      |  2 +-
 .../text_generation_server/models/__init__.py | 18 +++---
 5 files changed, 70 insertions(+), 15 deletions(-)

diff --git a/router/src/infer/chat_template.rs b/router/src/infer/chat_template.rs
index 7c2753ed..ef4beee2 100644
--- a/router/src/infer/chat_template.rs
+++ b/router/src/infer/chat_template.rs
@@ -1,3 +1,5 @@
+use std::collections::HashSet;
+
 use crate::infer::InferError;
 use crate::{
     ChatTemplateInputs, GrammarType, Message, MessageChunk, TextMessage, TokenizerConfigToken,
@@ -16,6 +18,7 @@ pub(crate) struct ChatTemplate {
     bos_token: Option<String>,
     eos_token: Option<String>,
     use_default_tool_template: bool,
+    variables: HashSet<String>,
 }
 
 impl ChatTemplate {
@@ -30,19 +33,22 @@ impl ChatTemplate {
         let template_str = template.into_boxed_str();
         env.add_function("raise_exception", raise_exception);
 
-        // check if contains the tools variable within the template
-        let use_default_tool_template =
-            !template_str.as_ref().replace(' ', "").contains("{{tools}}");
         // leaking env and template_str as read-only, static resources for performance.
         let template = Box::leak(env)
             .template_from_str(Box::leak(template_str))
             .unwrap();
 
+        // get the list of variables that are used in the template
+        let variables = template.undeclared_variables(true);
+        // check if the `tools` variable is used in the template
+        let use_default_tool_template = !variables.contains("tools");
+
         Self {
             template,
             bos_token: bos_token.map(|token| token.as_str().to_string()),
             eos_token: eos_token.map(|token| token.as_str().to_string()),
             use_default_tool_template,
+            variables,
         }
     }
 
@@ -64,6 +70,11 @@ impl ChatTemplate {
 
         let messages: Vec<TextMessage> = messages.into_iter().map(|c| c.into()).collect();
 
+        // check if guideline is expected but not provided
+        if self.variables.contains("guideline") && guideline.is_none() {
+            return Err(InferError::MissingTemplateVariable("guideline".to_string()));
+        }
+
         self.template
             .render(ChatTemplateInputs {
                 guideline,
@@ -82,7 +93,8 @@ impl ChatTemplate {
 #[cfg(test)]
 mod tests {
     use crate::infer::chat_template::raise_exception;
-    use crate::{ChatTemplateInputs, TextMessage};
+    use crate::infer::ChatTemplate;
+    use crate::{ChatTemplateInputs, Message, MessageContent, TextMessage, TokenizerConfigToken};
     use minijinja::Environment;
 
     #[test]
@@ -770,4 +782,45 @@ mod tests {
             assert_eq!(result, target);
         }
     }
+
+    #[test]
+    fn test_chat_template_invalid_with_guideline() {
+        let ct = ChatTemplate::new(
+            "{{- bos_token }}\n{%- if messages[-1].role == 'user' %}\n    {{- \"You are a policy expert trying to help determine whether a user prompt is in violation of the defined safety policies.\\n\\n\" }}\n    {{- \"<start_of_turn>\\n\" }}\n    {{- \"Human Question: \" + messages[-1].content }}\n    {{- \"\\n<end_of_turn>\\n\\n\" }}\n    {{- \"Our safety principle is defined in the below:\\n\\n\" }}\n    {{- \"* \" + guideline + \"\\n\" }}\n    {{- \"\\n===\\n\\n\" }}\n    {{- \"Does the human question violate the above principle? Your answer must start with 'Yes' or 'No'. And then walk through step by step to be sure we answer correctly.\\n\\n\" }}\n{%- else %}\n    {{- \"You are a policy expert trying to help determine whether an AI response to a prompt is in violation of the defined safety policies.\\n\\n\" }}\n    {{- \"<start_of_turn>\\n\" }}\n    {{- \"Human Question: \" + messages[-2].content }}\n    {{- \"\\n<end_of_turn>\\n\" }}\n    {{- \"<start_of_turn>\\n\" }}\n    {{- \"Chatbot Response: \" + messages[-1].content }}\n    {{- \"\\n<end_of_turn>\\n\\n\" }}\n    {{- \"Our safety principle is defined in the below:\\n\\n\" }}\n    {{- \"* \" + guideline + \"\\n\" }}\n    {{- \"\\n===\\n\\n\" }}\n    {{- \"Does the Chatbot Response violate the above principle? Your answer must  start with 'Yes' or 'No'. And then walk through step by step to be sure we answer correctly.\\n\\n\" }}\n{%- endif %}\n\n".to_string(),
+            Some(TokenizerConfigToken::String("<s>".to_string())),
+            Some(TokenizerConfigToken::String("</s>".to_string())),
+        );
+
+        // convert TextMessage to Message
+        let msgs: Vec<Message> = vec![
+            Message {
+                name: None,
+                role: "user".to_string(),
+                content: MessageContent::SingleText(
+                    "I'd like to show off how chat templating works!".to_string(),
+                ),
+            },
+            Message {
+                name: None,
+                role: "assistant".to_string(),
+                content: MessageContent::SingleText(
+                    "I'm doing great. How can I help you today?".to_string(),
+                ),
+            },
+            Message {
+                name: None,
+                role: "user".to_string(),
+                content: MessageContent::SingleText("Hello, how are you?".to_string()),
+            },
+        ];
+
+        let result = ct.apply(None, msgs, None);
+
+        match result {
+            Ok(_) => panic!("Should have failed since no guideline is provided"),
+            Err(e) => {
+                assert_eq!(e.to_string(), "Missing template vatiable: guideline")
+            }
+        }
+    }
 }
diff --git a/router/src/infer/mod.rs b/router/src/infer/mod.rs
index 58d5cf9a..c9354d9a 100644
--- a/router/src/infer/mod.rs
+++ b/router/src/infer/mod.rs
@@ -337,6 +337,8 @@ pub enum InferError {
     IncompleteGeneration,
     #[error("Template error: {0}")]
     TemplateError(#[from] minijinja::Error),
+    #[error("Missing template vatiable: {0}")]
+    MissingTemplateVariable(String),
     #[error("Tool error: {0}")]
     ToolError(String),
 }
@@ -349,6 +351,7 @@ impl InferError {
             InferError::ValidationError(_) => "validation",
             InferError::IncompleteGeneration => "incomplete_generation",
             InferError::TemplateError(_) => "template_error",
+            InferError::MissingTemplateVariable(_) => "missing_template_variable",
             InferError::ToolError(_) => "tool_error",
         }
     }
diff --git a/router/src/server.rs b/router/src/server.rs
index 8c0bd762..99ec077f 100644
--- a/router/src/server.rs
+++ b/router/src/server.rs
@@ -2297,6 +2297,7 @@ impl From<InferError> for (StatusCode, Json<ErrorResponse>) {
             InferError::ValidationError(_) => StatusCode::UNPROCESSABLE_ENTITY,
             InferError::IncompleteGeneration => StatusCode::INTERNAL_SERVER_ERROR,
             InferError::TemplateError(_) => StatusCode::UNPROCESSABLE_ENTITY,
+            InferError::MissingTemplateVariable(_) => StatusCode::UNPROCESSABLE_ENTITY,
             InferError::ToolError(_) => StatusCode::UNPROCESSABLE_ENTITY,
         };
 
diff --git a/router/src/validation.rs b/router/src/validation.rs
index 5011158a..0024723c 100644
--- a/router/src/validation.rs
+++ b/router/src/validation.rs
@@ -830,7 +830,7 @@ mod tests {
             .await
         {
             // Err(ValidationError::MaxNewTokens(1, 10)) => (),
-            Ok((_s, 0, 10)) => (),
+            Ok((_s, _, 0, 10)) => (),
             r => panic!("Unexpected not max new tokens: {r:?}"),
         }
     }
diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py
index 960b426b..4fa9e66d 100644
--- a/server/text_generation_server/models/__init__.py
+++ b/server/text_generation_server/models/__init__.py
@@ -497,17 +497,15 @@ def get_model(
         else -1
     )
 
-    if max_input_tokens is not None and max_input_tokens <= sliding_window:
-        sliding_window = -1
+    should_use_sliding_window = (
+        sliding_window is not None and sliding_window != -1 and SUPPORTS_WINDOWING
+    )
 
-    if (
-        (sliding_window is not None and sliding_window != -1)
-        and not SUPPORTS_WINDOWING
-        and max_input_tokens > sliding_window
-    ):
-        raise ValueError(
-            f"The backend {SYSTEM} does not support sliding window attention that is used by the model type {model_type}. To use this model nonetheless with the {SYSTEM} backend, please launch TGI with the argument `--max-input-tokens` smaller than sliding_window={sliding_window} (got here max_input_tokens={max_input_tokens})."
-        )
+    if should_use_sliding_window:
+        if max_input_tokens is not None and max_input_tokens > sliding_window:
+            raise ValueError(
+                f"The backend {SYSTEM} does not support sliding window attention that is used by the model type {model_type}. To use this model nonetheless with the {SYSTEM} backend, please launch TGI with the argument `--max-input-tokens` smaller than sliding_window={sliding_window} (got here max_input_tokens={max_input_tokens})."
+            )
 
     if model_type == DEEPSEEK_V2:
         if FLASH_ATTENTION:

From 4c3f8a70a1c8590851aa3d7c82a7cabf01ed6e87 Mon Sep 17 00:00:00 2001
From: drbh <david.richard.holtz@gmail.com>
Date: Mon, 12 Aug 2024 11:24:32 -0400
Subject: [PATCH 35/72] fix: allocate tmp based on sgmv kernel if available
 (#2345)

* fix: allocate tmp based on sgmv kernel if available

* fix: re add copy build artifacts step for punica kernels
---
 Dockerfile                                  |  2 ++
 server/text_generation_server/utils/sgmv.py | 12 ++++++++----
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index c68f76f6..458ff699 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -226,6 +226,8 @@ COPY --from=exllamav2-kernels-builder /usr/src/build/lib.linux-x86_64-cpython-31
 COPY --from=awq-kernels-builder /usr/src/llm-awq/awq/kernels/build/lib.linux-x86_64-cpython-310 /opt/conda/lib/python3.10/site-packages
 # Copy build artifacts from eetq kernels builder
 COPY --from=eetq-kernels-builder /usr/src/eetq/build/lib.linux-x86_64-cpython-310 /opt/conda/lib/python3.10/site-packages
+# Copy build artifacts from lorax punica kernels builder
+COPY --from=lorax-punica-builder /usr/src/lorax-punica/server/punica_kernels/build/lib.linux-x86_64-cpython-310 /opt/conda/lib/python3.10/site-packages
 # Copy build artifacts from fbgemm builder
 COPY --from=fbgemm-builder /usr/src/fbgemm/fbgemm_gpu/_skbuild/linux-x86_64-3.10/cmake-install /opt/conda/lib/python3.10/site-packages
 # Copy build artifacts from vllm builder
diff --git a/server/text_generation_server/utils/sgmv.py b/server/text_generation_server/utils/sgmv.py
index e0aec25f..2d0a73a5 100644
--- a/server/text_generation_server/utils/sgmv.py
+++ b/server/text_generation_server/utils/sgmv.py
@@ -151,13 +151,17 @@ def get_tmp_expand_size(size: int) -> int:
 def get_tmp_tensors(
     nsegments: int, lora_rank: int, device: torch.device
 ) -> Tuple[torch.Tensor, torch.Tensor]:
-    if use_cutlass_shrink(lora_rank) and has_sgmv():
+    use_cutlass = use_cutlass_shrink(lora_rank) and has_sgmv()
+    has_sgmv_available = has_sgmv()
+
+    if use_cutlass:
         tmp = get_tmp_tensor_for_size(nsegments, device)
         return tmp, tmp
+    elif has_sgmv_available:
+        return get_tmp_tensor(device), get_tmp_tensor_for_size(nsegments, device)
     else:
-        tmp_shrink = get_tmp_tensor(device)
-        tmp_expand = get_tmp_tensor_for_size_no_kernels(nsegments, device)
-        return tmp_shrink, tmp_expand
+        tmp = get_tmp_tensor_for_size(nsegments, device)
+        return tmp, tmp
 
 
 def lora_a_sgmv_cutlass(

From 30395b09f4eff271dd1dfdc49be4fd46f4a546dd Mon Sep 17 00:00:00 2001
From: drbh <david.richard.holtz@gmail.com>
Date: Mon, 12 Aug 2024 11:26:11 -0400
Subject: [PATCH 36/72] fix: improve completions to send a final chunk with
 usage details (#2336)

* fix: improve completions to send a final chunk with usage details

* fix: include finish reason string

* fix: remove dev debug trait and unneeded mut

* fix: update openapi schema
---
 docs/openapi.json    |  9 ++++++++-
 router/src/lib.rs    |  2 ++
 router/src/server.rs | 42 ++++++++++++++++++++++++++++++++++--------
 3 files changed, 44 insertions(+), 9 deletions(-)

diff --git a/docs/openapi.json b/docs/openapi.json
index ecd56e4d..df21e19d 100644
--- a/docs/openapi.json
+++ b/docs/openapi.json
@@ -1824,7 +1824,8 @@
         "type": "object",
         "required": [
           "finish_reason",
-          "generated_tokens"
+          "generated_tokens",
+          "input_length"
         ],
         "properties": {
           "finish_reason": {
@@ -1836,6 +1837,12 @@
             "example": 1,
             "minimum": 0
           },
+          "input_length": {
+            "type": "integer",
+            "format": "int32",
+            "example": 1,
+            "minimum": 0
+          },
           "seed": {
             "type": "integer",
             "format": "int64",
diff --git a/router/src/lib.rs b/router/src/lib.rs
index 0a15c495..d7eb4475 100644
--- a/router/src/lib.rs
+++ b/router/src/lib.rs
@@ -1219,6 +1219,8 @@ pub(crate) struct StreamDetails {
     pub generated_tokens: u32,
     #[schema(nullable = true, example = 42)]
     pub seed: Option<u64>,
+    #[schema(example = 1)]
+    pub input_length: u32,
 }
 
 #[derive(Serialize, ToSchema)]
diff --git a/router/src/server.rs b/router/src/server.rs
index 99ec077f..ab268efa 100644
--- a/router/src/server.rs
+++ b/router/src/server.rs
@@ -533,7 +533,7 @@ async fn generate_stream_internal(
         } else {
             match infer.generate_stream(req).instrument(info_span!(parent: &span, "async_stream")).await {
                 // Keep permit as long as generate_stream lives
-                Ok((_permit, _input_length, response_stream)) => {
+                Ok((_permit, input_length, response_stream)) => {
                     let mut index = 0;
                     let mut response_stream = Box::pin(response_stream);
                     // Server-Sent Event stream
@@ -576,6 +576,7 @@ async fn generate_stream_internal(
                                                 finish_reason: generated_text.finish_reason,
                                                 generated_tokens: generated_text.generated_tokens,
                                                 seed: generated_text.seed,
+                                                input_length,
                                             }),
                                             false => None,
                                         };
@@ -801,21 +802,46 @@ async fn completions(
                         .unwrap_or_else(|_| std::time::Duration::from_secs(0))
                         .as_secs();
 
-                    event
-                        .json_data(Completion::Chunk(Chunk {
-                            id: "".to_string(),
-                            created: current_time,
+                    let message = match stream_token.details {
+                        Some(details) => {
+                            let completion_tokens = details.generated_tokens;
+                            let prompt_tokens = details.input_length;
+                            let total_tokens = prompt_tokens + completion_tokens;
 
+                            Completion::Final(CompletionFinal {
+                                id: String::new(),
+                                created: current_time,
+                                model: model_id.clone(),
+                                system_fingerprint: system_fingerprint.clone(),
+                                choices: vec![CompletionComplete {
+                                    finish_reason: details.finish_reason.to_string(),
+                                    index: index as u32,
+                                    logprobs: None,
+                                    text: stream_token.token.text,
+                                }],
+                                usage: Usage {
+                                    prompt_tokens,
+                                    completion_tokens,
+                                    total_tokens,
+                                },
+                            })
+                        }
+                        None => Completion::Chunk(Chunk {
+                            id: String::new(),
+                            created: current_time,
                             choices: vec![CompletionComplete {
-                                finish_reason: "".to_string(),
+                                finish_reason: String::new(),
                                 index: index as u32,
                                 logprobs: None,
                                 text: stream_token.token.text,
                             }],
-
                             model: model_id.clone(),
                             system_fingerprint: system_fingerprint.clone(),
-                        }))
+                        }),
+                    };
+
+                    event
+                        .json_data(message)
                         .unwrap_or_else(|_e| Event::default())
                 };
 

From 19ea85f8dc7d8d29ddfefa7f906a16747e9996b9 Mon Sep 17 00:00:00 2001
From: Nicolas Patry <patry.nicolas@protonmail.com>
Date: Mon, 12 Aug 2024 18:09:16 +0200
Subject: [PATCH 37/72] Updating the flake. (#2404)

---
 _server.nix |  12 ++++
 flake.lock  | 158 +++++++++++++++++++++++++++++++++++++++++++++++-----
 flake.nix   |  10 ++++
 3 files changed, 165 insertions(+), 15 deletions(-)
 create mode 100644 _server.nix

diff --git a/_server.nix b/_server.nix
new file mode 100644
index 00000000..bc11ba6b
--- /dev/null
+++ b/_server.nix
@@ -0,0 +1,12 @@
+{ mkPoetryApplication, pkg-config, protobuf, openssl }:
+
+mkPoetryApplication {
+  # name = "text-generation-server";
+
+  projectDir = ./server;
+
+  # nativeBuildInputs = [ pkg-config ];
+
+  # buildInputs = [ openssl.dev protobuf ];
+
+}
diff --git a/flake.lock b/flake.lock
index 7889e7cf..47f14626 100644
--- a/flake.lock
+++ b/flake.lock
@@ -33,19 +33,96 @@
         "type": "github"
       }
     },
-    "nixpkgs": {
+    "flake-utils_2": {
+      "inputs": {
+        "systems": "systems_2"
+      },
       "locked": {
-        "lastModified": 1723099294,
-        "narHash": "sha256-kkijy6sXo/SOhFw7ZEfYHbj1FJHxoeetOVOn3qNHc5o=",
-        "owner": "danieldk",
-        "repo": "nixpkgs",
-        "rev": "45892b6ec142eaf300d777926983a433b5842c88",
+        "lastModified": 1710146030,
+        "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a",
         "type": "github"
       },
       "original": {
-        "owner": "danieldk",
-        "ref": "cudnn-9.3",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "nix-github-actions": {
+      "inputs": {
+        "nixpkgs": [
+          "poetry2nix",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1703863825,
+        "narHash": "sha256-rXwqjtwiGKJheXB43ybM8NwWB8rO2dSRrEqes0S7F5Y=",
+        "owner": "nix-community",
+        "repo": "nix-github-actions",
+        "rev": "5163432afc817cf8bd1f031418d1869e4c9d5547",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "nix-github-actions",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1719763542,
+        "narHash": "sha256-mXkOj9sJ0f69Nkc2dGGOWtof9d1YNY8Le/Hia3RN+8Q=",
+        "owner": "NixOS",
         "repo": "nixpkgs",
+        "rev": "e6cdd8a11b26b4d60593733106042141756b54a3",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-unstable-small",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs_2": {
+      "locked": {
+        "lastModified": 1723418128,
+        "narHash": "sha256-k1pEqsnB6ikZyasXbtV6A9akPZMKlsyENPDUA6PXoJo=",
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "rev": "129f579cbb5b4c1ad258fd96bdfb78eb14802727",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nixos",
+        "ref": "nixos-unstable-small",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "poetry2nix": {
+      "inputs": {
+        "flake-utils": "flake-utils_2",
+        "nix-github-actions": "nix-github-actions",
+        "nixpkgs": "nixpkgs",
+        "systems": "systems_3",
+        "treefmt-nix": "treefmt-nix"
+      },
+      "locked": {
+        "lastModified": 1723343306,
+        "narHash": "sha256-/6sRkPq7/5weX2y0V8sQ29Sz35nt8kyj+BsFtkhgbJE=",
+        "owner": "nix-community",
+        "repo": "poetry2nix",
+        "rev": "4a1c112ff0c67f496573dc345bd0b2247818fc29",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "poetry2nix",
         "type": "github"
       }
     },
@@ -56,6 +133,7 @@
           "tgi-nix",
           "nixpkgs"
         ],
+        "poetry2nix": "poetry2nix",
         "rust-overlay": "rust-overlay",
         "tgi-nix": "tgi-nix"
       }
@@ -68,11 +146,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1723170066,
-        "narHash": "sha256-SFkQfOA+8AIYJsPlQtxNP+z5jRLfz91z/aOrV94pPmw=",
+        "lastModified": 1723429325,
+        "narHash": "sha256-4x/32xTCd+xCwFoI/kKSiCr5LQA2ZlyTRYXKEni5HR8=",
         "owner": "oxalica",
         "repo": "rust-overlay",
-        "rev": "fecfe4d7c96fea2982c7907997b387a6b52c1093",
+        "rev": "65e3dc0fe079fe8df087cd38f1fe6836a0373aad",
         "type": "github"
       },
       "original": {
@@ -96,17 +174,46 @@
         "type": "github"
       }
     },
+    "systems_2": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
+    "systems_3": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "id": "systems",
+        "type": "indirect"
+      }
+    },
     "tgi-nix": {
       "inputs": {
         "flake-compat": "flake-compat",
-        "nixpkgs": "nixpkgs"
+        "nixpkgs": "nixpkgs_2"
       },
       "locked": {
-        "lastModified": 1723234585,
-        "narHash": "sha256-HChJpNP155FPhHr9C5BtqllV8Uv/Ebg59HhMc/HhQrc=",
+        "lastModified": 1723450799,
+        "narHash": "sha256-cuT/ce7R2D5Lx6Ted4YS4y+WrAAOXQFAbzLsM1vtPo8=",
         "owner": "danieldk",
         "repo": "tgi-nix",
-        "rev": "15bd4a978d6c2c8b04b7afc335d137dbe41e73df",
+        "rev": "29f9b45bd613eced65c4a5241a00aa9346f63d90",
         "type": "github"
       },
       "original": {
@@ -114,6 +221,27 @@
         "repo": "tgi-nix",
         "type": "github"
       }
+    },
+    "treefmt-nix": {
+      "inputs": {
+        "nixpkgs": [
+          "poetry2nix",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1719749022,
+        "narHash": "sha256-ddPKHcqaKCIFSFc/cvxS14goUhCOAwsM1PbMr0ZtHMg=",
+        "owner": "numtide",
+        "repo": "treefmt-nix",
+        "rev": "8df5ff62195d4e67e2264df0b7f5e8c9995fd0bd",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "treefmt-nix",
+        "type": "github"
+      }
     }
   },
   "root": "root",
diff --git a/flake.nix b/flake.nix
index 761c4af8..1f842e8e 100644
--- a/flake.nix
+++ b/flake.nix
@@ -3,6 +3,7 @@
     tgi-nix.url = "github:danieldk/tgi-nix";
     nixpkgs.follows = "tgi-nix/nixpkgs";
     flake-utils.url = "github:numtide/flake-utils";
+    poetry2nix.url = "github:nix-community/poetry2nix";
     rust-overlay = {
       url = "github:oxalica/rust-overlay";
       inputs.nixpkgs.follows = "tgi-nix/nixpkgs";
@@ -15,6 +16,7 @@
       flake-utils,
       rust-overlay,
       tgi-nix,
+      poetry2nix,
     }:
     flake-utils.lib.eachDefaultSystem (
       system:
@@ -30,6 +32,11 @@
             tgi-nix.overlay
           ];
         };
+
+        inherit (poetry2nix.lib.mkPoetry2Nix { inherit pkgs; }) mkPoetryEditablePackage;
+        text-generation-server = mkPoetryEditablePackage {
+            editablePackageSources = ./server;
+        };
       in
       {
         devShells.default =
@@ -50,14 +57,17 @@
                 venvShellHook
                 pip
 
+                click
                 einops
                 fbgemm-gpu
+                flashinfer
                 flash-attn
                 flash-attn-layer-norm
                 flash-attn-rotary
                 grpc-interceptor
                 grpcio-reflection
                 grpcio-status
+                grpcio-tools
                 hf-transfer
                 loguru
                 marlin-kernels

From 9a7830bd287fb1ca2a3e94f208acd2eb881eb311 Mon Sep 17 00:00:00 2001
From: drbh <david.richard.holtz@gmail.com>
Date: Mon, 12 Aug 2024 14:38:59 -0400
Subject: [PATCH 38/72] Pr 2395 ci run (#2406)

* fix(router): Fix appending to message content

* feat: add message and chat template test

---------

Co-authored-by: Simone Rossi <simone.rossi.93@gmail.com>
---
 router/src/infer/chat_template.rs | 40 ++++++++++++++++++++++++++++++-
 router/src/lib.rs                 | 35 +++++++++++++++++++++++++--
 2 files changed, 72 insertions(+), 3 deletions(-)

diff --git a/router/src/infer/chat_template.rs b/router/src/infer/chat_template.rs
index ef4beee2..a8537818 100644
--- a/router/src/infer/chat_template.rs
+++ b/router/src/infer/chat_template.rs
@@ -94,7 +94,9 @@ impl ChatTemplate {
 mod tests {
     use crate::infer::chat_template::raise_exception;
     use crate::infer::ChatTemplate;
-    use crate::{ChatTemplateInputs, Message, MessageContent, TextMessage, TokenizerConfigToken};
+    use crate::{
+        ChatTemplateInputs, GrammarType, Message, MessageContent, TextMessage, TokenizerConfigToken,
+    };
     use minijinja::Environment;
 
     #[test]
@@ -823,4 +825,40 @@ mod tests {
             }
         }
     }
+
+    #[test]
+    fn test_chat_template_with_default_tool_template() {
+        let ct = ChatTemplate::new(
+            "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token + ' ' }}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %}".to_string(),
+            Some(TokenizerConfigToken::String("<s>".to_string())),
+            Some(TokenizerConfigToken::String("</s>".to_string())),
+        );
+
+        // convert TextMessage to Message
+        let msgs: Vec<Message> = vec![
+            Message {
+                name: None,
+                role: "user".to_string(),
+                content: MessageContent::SingleText(
+                    "I'd like to show off how chat templating works!".to_string(),
+                ),
+            },
+            Message {
+                name: None,
+                role: "assistant".to_string(),
+                content: MessageContent::SingleText("Great! How can I help you today?".to_string()),
+            },
+            Message {
+                name: None,
+                role: "user".to_string(),
+                content: MessageContent::SingleText("Just testing".to_string()),
+            },
+        ];
+        let tools = serde_json::json!("[]");
+        let tool_prompt = "This default prompt will be used".to_string();
+        let grammer_with_prompt = (GrammarType::Json(tools), tool_prompt);
+        let result = ct.apply(None, msgs, Some(grammer_with_prompt));
+        let expected = "<s>[INST] I'd like to show off how chat templating works! [/INST]Great! How can I help you today?</s> [INST] Just testing\n---\nThis default prompt will be used\n\"[]\" [/INST]".to_string();
+        assert_eq!(result.unwrap(), expected);
+    }
 }
diff --git a/router/src/lib.rs b/router/src/lib.rs
index d7eb4475..1b2ff153 100644
--- a/router/src/lib.rs
+++ b/router/src/lib.rs
@@ -1016,8 +1016,10 @@ impl MessageContent {
     pub fn push(&mut self, chunk: MessageChunk) {
         match self {
             MessageContent::SingleText(text) => {
-                *self =
-                    MessageContent::MultipleChunks(vec![MessageChunk::Text { text: text.clone() }]);
+                *self = MessageContent::MultipleChunks(vec![
+                    MessageChunk::Text { text: text.clone() },
+                    chunk,
+                ]);
             }
             MessageContent::MultipleChunks(chunks) => {
                 chunks.push(chunk);
@@ -1348,6 +1350,35 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_message_content_append() {
+        let mut content = MessageContent::SingleText("Initial text".to_string());
+        let chunk = MessageChunk::Text {
+            text: "Additional text".to_string(),
+        };
+
+        content.push(chunk);
+
+        match content {
+            MessageContent::MultipleChunks(chunks) => {
+                assert_eq!(chunks.len(), 2);
+                assert_eq!(
+                    chunks[0],
+                    MessageChunk::Text {
+                        text: "Initial text".to_string()
+                    }
+                );
+                assert_eq!(
+                    chunks[1],
+                    MessageChunk::Text {
+                        text: "Additional text".to_string()
+                    }
+                );
+            }
+            _ => panic!("Expected MultipleChunks, but got a different variant"),
+        }
+    }
+
     #[test]
     fn test_chat_request() {
         let json = json!({

From 8a7749b8fb678934a53ee72102644572a8d7b0a6 Mon Sep 17 00:00:00 2001
From: drbh <david.richard.holtz@gmail.com>
Date: Mon, 12 Aug 2024 17:59:37 -0400
Subject: [PATCH 39/72] fix: include create_exllama_buffers and set_device for
 exllama (#2407)

---
 server/text_generation_server/layers/gptq/__init__.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/server/text_generation_server/layers/gptq/__init__.py b/server/text_generation_server/layers/gptq/__init__.py
index 9c9b69d1..505caa59 100644
--- a/server/text_generation_server/layers/gptq/__init__.py
+++ b/server/text_generation_server/layers/gptq/__init__.py
@@ -422,12 +422,16 @@ elif CAN_EXLLAMA:
         if V2:
             from text_generation_server.layers.gptq.exllamav2 import (
                 QuantLinear as ExllamaQuantLinear,  # noqa: F401
+                create_exllama_buffers,  # noqa: F401
+                set_device,  # noqa: F401
             )
 
             HAS_EXLLAMA = "2"
         else:
             from text_generation_server.layers.gptq.exllama import (
                 Ex4bitLinear as ExllamaQuantLinear,  # noqa: F401
+                create_exllama_buffers,  # noqa: F401
+                set_device,  # noqa: F401
             )
 
             HAS_EXLLAMA = "1"

From 6f4bb4f26ffa6351a7caa27d6e81fc4872b3a16b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Dani=C3=ABl=20de=20Kok?= <me@danieldk.eu>
Date: Tue, 13 Aug 2024 10:44:15 +0200
Subject: [PATCH 40/72] nix: incremental build of the launcher (#2410)

---
 _launcher.nix |  18 --
 flake.lock    | 844 +++++++++++++++++++++++++++++++++++++++++++++++++-
 flake.nix     |  14 +-
 3 files changed, 846 insertions(+), 30 deletions(-)
 delete mode 100644 _launcher.nix

diff --git a/_launcher.nix b/_launcher.nix
deleted file mode 100644
index 1acae7e1..00000000
--- a/_launcher.nix
+++ /dev/null
@@ -1,18 +0,0 @@
-{ buildRustPackage, importCargoLock, pkg-config, protobuf, openssl }:
-
-buildRustPackage {
-  name = "text-generation-lancher";
-
-  src = ./.;
-
-  sourceDir = ./launcher;
-
-  cargoLock = {
-    lockFile = ./Cargo.lock;
-  };
-
-  nativeBuildInputs = [ pkg-config ];
-
-  buildInputs = [ openssl.dev protobuf ];
-
-}
diff --git a/flake.lock b/flake.lock
index 47f14626..bc114032 100644
--- a/flake.lock
+++ b/flake.lock
@@ -1,6 +1,309 @@
 {
   "nodes": {
+    "cachix": {
+      "inputs": {
+        "devenv": [
+          "crate2nix"
+        ],
+        "flake-compat": [
+          "crate2nix"
+        ],
+        "nixpkgs": "nixpkgs",
+        "pre-commit-hooks": [
+          "crate2nix"
+        ]
+      },
+      "locked": {
+        "lastModified": 1709700175,
+        "narHash": "sha256-A0/6ZjLmT9qdYzKHmevnEIC7G+GiZ4UCr8v0poRPzds=",
+        "owner": "cachix",
+        "repo": "cachix",
+        "rev": "be97b37989f11b724197b5f4c7ffd78f12c8c4bf",
+        "type": "github"
+      },
+      "original": {
+        "owner": "cachix",
+        "ref": "latest",
+        "repo": "cachix",
+        "type": "github"
+      }
+    },
+    "cachix_2": {
+      "inputs": {
+        "devenv": [
+          "crate2nix",
+          "crate2nix_stable"
+        ],
+        "flake-compat": [
+          "crate2nix",
+          "crate2nix_stable"
+        ],
+        "nixpkgs": "nixpkgs_2",
+        "pre-commit-hooks": [
+          "crate2nix",
+          "crate2nix_stable"
+        ]
+      },
+      "locked": {
+        "lastModified": 1716549461,
+        "narHash": "sha256-lHy5kgx6J8uD+16SO47dPrbob98sh+W1tf4ceSqPVK4=",
+        "owner": "cachix",
+        "repo": "cachix",
+        "rev": "e2bb269fb8c0828d5d4d2d7b8d09ea85abcacbd4",
+        "type": "github"
+      },
+      "original": {
+        "owner": "cachix",
+        "ref": "latest",
+        "repo": "cachix",
+        "type": "github"
+      }
+    },
+    "cachix_3": {
+      "inputs": {
+        "devenv": [
+          "crate2nix",
+          "crate2nix_stable",
+          "crate2nix_stable"
+        ],
+        "flake-compat": [
+          "crate2nix",
+          "crate2nix_stable",
+          "crate2nix_stable"
+        ],
+        "nixpkgs": "nixpkgs_3",
+        "pre-commit-hooks": [
+          "crate2nix",
+          "crate2nix_stable",
+          "crate2nix_stable"
+        ]
+      },
+      "locked": {
+        "lastModified": 1716549461,
+        "narHash": "sha256-lHy5kgx6J8uD+16SO47dPrbob98sh+W1tf4ceSqPVK4=",
+        "owner": "cachix",
+        "repo": "cachix",
+        "rev": "e2bb269fb8c0828d5d4d2d7b8d09ea85abcacbd4",
+        "type": "github"
+      },
+      "original": {
+        "owner": "cachix",
+        "ref": "latest",
+        "repo": "cachix",
+        "type": "github"
+      }
+    },
+    "crate2nix": {
+      "inputs": {
+        "cachix": "cachix",
+        "crate2nix_stable": "crate2nix_stable",
+        "devshell": "devshell_3",
+        "flake-compat": "flake-compat_3",
+        "flake-parts": "flake-parts_3",
+        "nix-test-runner": "nix-test-runner_3",
+        "nixpkgs": [
+          "tgi-nix",
+          "nixpkgs"
+        ],
+        "pre-commit-hooks": "pre-commit-hooks_3"
+      },
+      "locked": {
+        "lastModified": 1723311214,
+        "narHash": "sha256-xdGZQBEa1AC2us/sY3igS/CucWY6jErXsAvCFRhB2LI=",
+        "owner": "nix-community",
+        "repo": "crate2nix",
+        "rev": "236f6addfd452a48be805819e3216af79e988fd5",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "crate2nix",
+        "type": "github"
+      }
+    },
+    "crate2nix_stable": {
+      "inputs": {
+        "cachix": "cachix_2",
+        "crate2nix_stable": "crate2nix_stable_2",
+        "devshell": "devshell_2",
+        "flake-compat": "flake-compat_2",
+        "flake-parts": "flake-parts_2",
+        "nix-test-runner": "nix-test-runner_2",
+        "nixpkgs": "nixpkgs_5",
+        "pre-commit-hooks": "pre-commit-hooks_2"
+      },
+      "locked": {
+        "lastModified": 1719760004,
+        "narHash": "sha256-esWhRnt7FhiYq0CcIxw9pvH+ybOQmWBfHYMtleaMhBE=",
+        "owner": "nix-community",
+        "repo": "crate2nix",
+        "rev": "1dee214bb20855fa3e1e7bb98d28922ddaff8c57",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "ref": "0.14.1",
+        "repo": "crate2nix",
+        "type": "github"
+      }
+    },
+    "crate2nix_stable_2": {
+      "inputs": {
+        "cachix": "cachix_3",
+        "crate2nix_stable": "crate2nix_stable_3",
+        "devshell": "devshell",
+        "flake-compat": "flake-compat",
+        "flake-parts": "flake-parts",
+        "nix-test-runner": "nix-test-runner",
+        "nixpkgs": "nixpkgs_4",
+        "pre-commit-hooks": "pre-commit-hooks"
+      },
+      "locked": {
+        "lastModified": 1712821484,
+        "narHash": "sha256-rGT3CW64cJS9nlnWPFWSc1iEa3dNZecVVuPVGzcsHe8=",
+        "owner": "nix-community",
+        "repo": "crate2nix",
+        "rev": "42883afcad3823fa5811e967fb7bff54bc3c9d6d",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "ref": "0.14.0",
+        "repo": "crate2nix",
+        "type": "github"
+      }
+    },
+    "crate2nix_stable_3": {
+      "inputs": {
+        "flake-utils": "flake-utils"
+      },
+      "locked": {
+        "lastModified": 1702842982,
+        "narHash": "sha256-A9AowkHIjsy1a4LuiPiVP88FMxyCWK41flZEZOUuwQM=",
+        "owner": "nix-community",
+        "repo": "crate2nix",
+        "rev": "75ac2973affa6b9b4f661a7b592cba6e4f51d426",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "ref": "0.12.0",
+        "repo": "crate2nix",
+        "type": "github"
+      }
+    },
+    "devshell": {
+      "inputs": {
+        "flake-utils": "flake-utils_2",
+        "nixpkgs": [
+          "crate2nix",
+          "crate2nix_stable",
+          "crate2nix_stable",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1717408969,
+        "narHash": "sha256-Q0OEFqe35fZbbRPPRdrjTUUChKVhhWXz3T9ZSKmaoVY=",
+        "owner": "numtide",
+        "repo": "devshell",
+        "rev": "1ebbe68d57457c8cae98145410b164b5477761f4",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "devshell",
+        "type": "github"
+      }
+    },
+    "devshell_2": {
+      "inputs": {
+        "flake-utils": "flake-utils_3",
+        "nixpkgs": [
+          "crate2nix",
+          "crate2nix_stable",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1717408969,
+        "narHash": "sha256-Q0OEFqe35fZbbRPPRdrjTUUChKVhhWXz3T9ZSKmaoVY=",
+        "owner": "numtide",
+        "repo": "devshell",
+        "rev": "1ebbe68d57457c8cae98145410b164b5477761f4",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "devshell",
+        "type": "github"
+      }
+    },
+    "devshell_3": {
+      "inputs": {
+        "flake-utils": "flake-utils_4",
+        "nixpkgs": [
+          "crate2nix",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1711099426,
+        "narHash": "sha256-HzpgM/wc3aqpnHJJ2oDqPBkNsqWbW0WfWUO8lKu8nGk=",
+        "owner": "numtide",
+        "repo": "devshell",
+        "rev": "2d45b54ca4a183f2fdcf4b19c895b64fbf620ee8",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "devshell",
+        "type": "github"
+      }
+    },
     "flake-compat": {
+      "locked": {
+        "lastModified": 1696426674,
+        "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=",
+        "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33",
+        "revCount": 57,
+        "type": "tarball",
+        "url": "https://api.flakehub.com/f/pinned/edolstra/flake-compat/1.0.1/018afb31-abd1-7bff-a5e4-cff7e18efb7a/source.tar.gz"
+      },
+      "original": {
+        "type": "tarball",
+        "url": "https://flakehub.com/f/edolstra/flake-compat/1.tar.gz"
+      }
+    },
+    "flake-compat_2": {
+      "locked": {
+        "lastModified": 1696426674,
+        "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=",
+        "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33",
+        "revCount": 57,
+        "type": "tarball",
+        "url": "https://api.flakehub.com/f/pinned/edolstra/flake-compat/1.0.1/018afb31-abd1-7bff-a5e4-cff7e18efb7a/source.tar.gz"
+      },
+      "original": {
+        "type": "tarball",
+        "url": "https://flakehub.com/f/edolstra/flake-compat/1.tar.gz"
+      }
+    },
+    "flake-compat_3": {
+      "locked": {
+        "lastModified": 1696426674,
+        "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=",
+        "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33",
+        "revCount": 57,
+        "type": "tarball",
+        "url": "https://api.flakehub.com/f/pinned/edolstra/flake-compat/1.0.1/018afb31-abd1-7bff-a5e4-cff7e18efb7a/source.tar.gz"
+      },
+      "original": {
+        "type": "tarball",
+        "url": "https://flakehub.com/f/edolstra/flake-compat/1.tar.gz"
+      }
+    },
+    "flake-compat_4": {
       "locked": {
         "lastModified": 1696426674,
         "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=",
@@ -15,10 +318,148 @@
         "type": "github"
       }
     },
+    "flake-parts": {
+      "inputs": {
+        "nixpkgs-lib": [
+          "crate2nix",
+          "crate2nix_stable",
+          "crate2nix_stable",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1719745305,
+        "narHash": "sha256-xwgjVUpqSviudEkpQnioeez1Uo2wzrsMaJKJClh+Bls=",
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "rev": "c3c5ecc05edc7dafba779c6c1a61cd08ac6583e9",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "type": "github"
+      }
+    },
+    "flake-parts_2": {
+      "inputs": {
+        "nixpkgs-lib": [
+          "crate2nix",
+          "crate2nix_stable",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1719745305,
+        "narHash": "sha256-xwgjVUpqSviudEkpQnioeez1Uo2wzrsMaJKJClh+Bls=",
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "rev": "c3c5ecc05edc7dafba779c6c1a61cd08ac6583e9",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "type": "github"
+      }
+    },
+    "flake-parts_3": {
+      "inputs": {
+        "nixpkgs-lib": [
+          "crate2nix",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1712014858,
+        "narHash": "sha256-sB4SWl2lX95bExY2gMFG5HIzvva5AVMJd4Igm+GpZNw=",
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "rev": "9126214d0a59633752a136528f5f3b9aa8565b7d",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "type": "github"
+      }
+    },
     "flake-utils": {
       "inputs": {
         "systems": "systems"
       },
+      "locked": {
+        "lastModified": 1694529238,
+        "narHash": "sha256-zsNZZGTGnMOf9YpHKJqMSsa0dXbfmxeoJ7xHlrt+xmY=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "ff7b65b44d01cf9ba6a71320833626af21126384",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "flake-utils_2": {
+      "inputs": {
+        "systems": "systems_2"
+      },
+      "locked": {
+        "lastModified": 1701680307,
+        "narHash": "sha256-kAuep2h5ajznlPMD9rnQyffWG8EM/C73lejGofXvdM8=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "4022d587cbbfd70fe950c1e2083a02621806a725",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "flake-utils_3": {
+      "inputs": {
+        "systems": "systems_3"
+      },
+      "locked": {
+        "lastModified": 1701680307,
+        "narHash": "sha256-kAuep2h5ajznlPMD9rnQyffWG8EM/C73lejGofXvdM8=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "4022d587cbbfd70fe950c1e2083a02621806a725",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "flake-utils_4": {
+      "inputs": {
+        "systems": "systems_4"
+      },
+      "locked": {
+        "lastModified": 1701680307,
+        "narHash": "sha256-kAuep2h5ajznlPMD9rnQyffWG8EM/C73lejGofXvdM8=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "4022d587cbbfd70fe950c1e2083a02621806a725",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "flake-utils_5": {
+      "inputs": {
+        "systems": "systems_5"
+      },
       "locked": {
         "lastModified": 1710146030,
         "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=",
@@ -33,9 +474,9 @@
         "type": "github"
       }
     },
-    "flake-utils_2": {
+    "flake-utils_6": {
       "inputs": {
-        "systems": "systems_2"
+        "systems": "systems_6"
       },
       "locked": {
         "lastModified": 1710146030,
@@ -51,6 +492,93 @@
         "type": "github"
       }
     },
+    "flake-utils_7": {
+      "inputs": {
+        "systems": "systems_7"
+      },
+      "locked": {
+        "lastModified": 1710146030,
+        "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "gitignore": {
+      "inputs": {
+        "nixpkgs": [
+          "crate2nix",
+          "crate2nix_stable",
+          "crate2nix_stable",
+          "pre-commit-hooks",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1709087332,
+        "narHash": "sha256-HG2cCnktfHsKV0s4XW83gU3F57gaTljL9KNSuG6bnQs=",
+        "owner": "hercules-ci",
+        "repo": "gitignore.nix",
+        "rev": "637db329424fd7e46cf4185293b9cc8c88c95394",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "gitignore.nix",
+        "type": "github"
+      }
+    },
+    "gitignore_2": {
+      "inputs": {
+        "nixpkgs": [
+          "crate2nix",
+          "crate2nix_stable",
+          "pre-commit-hooks",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1709087332,
+        "narHash": "sha256-HG2cCnktfHsKV0s4XW83gU3F57gaTljL9KNSuG6bnQs=",
+        "owner": "hercules-ci",
+        "repo": "gitignore.nix",
+        "rev": "637db329424fd7e46cf4185293b9cc8c88c95394",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "gitignore.nix",
+        "type": "github"
+      }
+    },
+    "gitignore_3": {
+      "inputs": {
+        "nixpkgs": [
+          "crate2nix",
+          "pre-commit-hooks",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1709087332,
+        "narHash": "sha256-HG2cCnktfHsKV0s4XW83gU3F57gaTljL9KNSuG6bnQs=",
+        "owner": "hercules-ci",
+        "repo": "gitignore.nix",
+        "rev": "637db329424fd7e46cf4185293b9cc8c88c95394",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "gitignore.nix",
+        "type": "github"
+      }
+    },
     "nix-github-actions": {
       "inputs": {
         "nixpkgs": [
@@ -72,7 +600,129 @@
         "type": "github"
       }
     },
+    "nix-test-runner": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1588761593,
+        "narHash": "sha256-FKJykltAN/g3eIceJl4SfDnnyuH2jHImhMrXS2KvGIs=",
+        "owner": "stoeffel",
+        "repo": "nix-test-runner",
+        "rev": "c45d45b11ecef3eb9d834c3b6304c05c49b06ca2",
+        "type": "github"
+      },
+      "original": {
+        "owner": "stoeffel",
+        "repo": "nix-test-runner",
+        "type": "github"
+      }
+    },
+    "nix-test-runner_2": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1588761593,
+        "narHash": "sha256-FKJykltAN/g3eIceJl4SfDnnyuH2jHImhMrXS2KvGIs=",
+        "owner": "stoeffel",
+        "repo": "nix-test-runner",
+        "rev": "c45d45b11ecef3eb9d834c3b6304c05c49b06ca2",
+        "type": "github"
+      },
+      "original": {
+        "owner": "stoeffel",
+        "repo": "nix-test-runner",
+        "type": "github"
+      }
+    },
+    "nix-test-runner_3": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1588761593,
+        "narHash": "sha256-FKJykltAN/g3eIceJl4SfDnnyuH2jHImhMrXS2KvGIs=",
+        "owner": "stoeffel",
+        "repo": "nix-test-runner",
+        "rev": "c45d45b11ecef3eb9d834c3b6304c05c49b06ca2",
+        "type": "github"
+      },
+      "original": {
+        "owner": "stoeffel",
+        "repo": "nix-test-runner",
+        "type": "github"
+      }
+    },
     "nixpkgs": {
+      "locked": {
+        "lastModified": 1700612854,
+        "narHash": "sha256-yrQ8osMD+vDLGFX7pcwsY/Qr5PUd6OmDMYJZzZi0+zc=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "19cbff58383a4ae384dea4d1d0c823d72b49d614",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs_2": {
+      "locked": {
+        "lastModified": 1715534503,
+        "narHash": "sha256-5ZSVkFadZbFP1THataCaSf0JH2cAH3S29hU9rrxTEqk=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "2057814051972fa1453ddfb0d98badbea9b83c06",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs_3": {
+      "locked": {
+        "lastModified": 1715534503,
+        "narHash": "sha256-5ZSVkFadZbFP1THataCaSf0JH2cAH3S29hU9rrxTEqk=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "2057814051972fa1453ddfb0d98badbea9b83c06",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs_4": {
+      "locked": {
+        "lastModified": 1719506693,
+        "narHash": "sha256-C8e9S7RzshSdHB7L+v9I51af1gDM5unhJ2xO1ywxNH8=",
+        "path": "/nix/store/4p0avw1s3vf27hspgqsrqs37gxk4i83i-source",
+        "rev": "b2852eb9365c6de48ffb0dc2c9562591f652242a",
+        "type": "path"
+      },
+      "original": {
+        "id": "nixpkgs",
+        "type": "indirect"
+      }
+    },
+    "nixpkgs_5": {
+      "locked": {
+        "lastModified": 1719506693,
+        "narHash": "sha256-C8e9S7RzshSdHB7L+v9I51af1gDM5unhJ2xO1ywxNH8=",
+        "path": "/nix/store/4p0avw1s3vf27hspgqsrqs37gxk4i83i-source",
+        "rev": "b2852eb9365c6de48ffb0dc2c9562591f652242a",
+        "type": "path"
+      },
+      "original": {
+        "id": "nixpkgs",
+        "type": "indirect"
+      }
+    },
+    "nixpkgs_6": {
       "locked": {
         "lastModified": 1719763542,
         "narHash": "sha256-mXkOj9sJ0f69Nkc2dGGOWtof9d1YNY8Le/Hia3RN+8Q=",
@@ -88,7 +738,7 @@
         "type": "github"
       }
     },
-    "nixpkgs_2": {
+    "nixpkgs_7": {
       "locked": {
         "lastModified": 1723418128,
         "narHash": "sha256-k1pEqsnB6ikZyasXbtV6A9akPZMKlsyENPDUA6PXoJo=",
@@ -106,10 +756,10 @@
     },
     "poetry2nix": {
       "inputs": {
-        "flake-utils": "flake-utils_2",
+        "flake-utils": "flake-utils_7",
         "nix-github-actions": "nix-github-actions",
-        "nixpkgs": "nixpkgs",
-        "systems": "systems_3",
+        "nixpkgs": "nixpkgs_6",
+        "systems": "systems_8",
         "treefmt-nix": "treefmt-nix"
       },
       "locked": {
@@ -126,9 +776,110 @@
         "type": "github"
       }
     },
+    "pre-commit-hooks": {
+      "inputs": {
+        "flake-compat": [
+          "crate2nix",
+          "crate2nix_stable",
+          "crate2nix_stable",
+          "flake-compat"
+        ],
+        "gitignore": "gitignore",
+        "nixpkgs": [
+          "crate2nix",
+          "crate2nix_stable",
+          "crate2nix_stable",
+          "nixpkgs"
+        ],
+        "nixpkgs-stable": [
+          "crate2nix",
+          "crate2nix_stable",
+          "crate2nix_stable",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1719259945,
+        "narHash": "sha256-F1h+XIsGKT9TkGO3omxDLEb/9jOOsI6NnzsXFsZhry4=",
+        "owner": "cachix",
+        "repo": "pre-commit-hooks.nix",
+        "rev": "0ff4381bbb8f7a52ca4a851660fc7a437a4c6e07",
+        "type": "github"
+      },
+      "original": {
+        "owner": "cachix",
+        "repo": "pre-commit-hooks.nix",
+        "type": "github"
+      }
+    },
+    "pre-commit-hooks_2": {
+      "inputs": {
+        "flake-compat": [
+          "crate2nix",
+          "crate2nix_stable",
+          "flake-compat"
+        ],
+        "gitignore": "gitignore_2",
+        "nixpkgs": [
+          "crate2nix",
+          "crate2nix_stable",
+          "nixpkgs"
+        ],
+        "nixpkgs-stable": [
+          "crate2nix",
+          "crate2nix_stable",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1719259945,
+        "narHash": "sha256-F1h+XIsGKT9TkGO3omxDLEb/9jOOsI6NnzsXFsZhry4=",
+        "owner": "cachix",
+        "repo": "pre-commit-hooks.nix",
+        "rev": "0ff4381bbb8f7a52ca4a851660fc7a437a4c6e07",
+        "type": "github"
+      },
+      "original": {
+        "owner": "cachix",
+        "repo": "pre-commit-hooks.nix",
+        "type": "github"
+      }
+    },
+    "pre-commit-hooks_3": {
+      "inputs": {
+        "flake-compat": [
+          "crate2nix",
+          "flake-compat"
+        ],
+        "flake-utils": "flake-utils_5",
+        "gitignore": "gitignore_3",
+        "nixpkgs": [
+          "crate2nix",
+          "nixpkgs"
+        ],
+        "nixpkgs-stable": [
+          "crate2nix",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1712055707,
+        "narHash": "sha256-4XLvuSIDZJGS17xEwSrNuJLL7UjDYKGJSbK1WWX2AK8=",
+        "owner": "cachix",
+        "repo": "pre-commit-hooks.nix",
+        "rev": "e35aed5fda3cc79f88ed7f1795021e559582093a",
+        "type": "github"
+      },
+      "original": {
+        "owner": "cachix",
+        "repo": "pre-commit-hooks.nix",
+        "type": "github"
+      }
+    },
     "root": {
       "inputs": {
-        "flake-utils": "flake-utils",
+        "crate2nix": "crate2nix",
+        "flake-utils": "flake-utils_6",
         "nixpkgs": [
           "tgi-nix",
           "nixpkgs"
@@ -190,6 +941,81 @@
       }
     },
     "systems_3": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
+    "systems_4": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
+    "systems_5": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
+    "systems_6": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
+    "systems_7": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
+    "systems_8": {
       "locked": {
         "lastModified": 1681028828,
         "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
@@ -205,8 +1031,8 @@
     },
     "tgi-nix": {
       "inputs": {
-        "flake-compat": "flake-compat",
-        "nixpkgs": "nixpkgs_2"
+        "flake-compat": "flake-compat_4",
+        "nixpkgs": "nixpkgs_7"
       },
       "locked": {
         "lastModified": 1723450799,
diff --git a/flake.nix b/flake.nix
index 1f842e8e..40b845b7 100644
--- a/flake.nix
+++ b/flake.nix
@@ -1,5 +1,9 @@
 {
   inputs = {
+    crate2nix = {
+      url = "github:nix-community/crate2nix";
+      inputs.nixpkgs.follows = "tgi-nix/nixpkgs";
+    };
     tgi-nix.url = "github:danieldk/tgi-nix";
     nixpkgs.follows = "tgi-nix/nixpkgs";
     flake-utils.url = "github:numtide/flake-utils";
@@ -12,6 +16,7 @@
   outputs =
     {
       self,
+      crate2nix,
       nixpkgs,
       flake-utils,
       rust-overlay,
@@ -21,6 +26,10 @@
     flake-utils.lib.eachDefaultSystem (
       system:
       let
+        cargoNix = crate2nix.tools.${system}.appliedCargoNix {
+          name = "tgi";
+          src = ./.;
+        };
         config = {
           allowUnfree = true;
           cudaSupport = true;
@@ -81,12 +90,11 @@
                 transformers
                 vllm
 
+                cargoNix.workspaceMembers.text-generation-launcher.build 
+
                 (callPackage ./router.nix {
                   inherit (rustPlatform) buildRustPackage importCargoLock;
                 })
-                (callPackage ./_launcher.nix {
-                  inherit (rustPlatform) buildRustPackage importCargoLock;
-                })
               ]);
 
             venvDir = "./.venv";

From cd9b15d17f0cc89bc443ff876b3bee762bb4856e Mon Sep 17 00:00:00 2001
From: Nicolas Patry <patry.nicolas@protonmail.com>
Date: Tue, 13 Aug 2024 10:49:18 +0200
Subject: [PATCH 41/72] Adding more kernels to flake. (#2411)

---
 flake.lock | 18 +++++++++---------
 flake.nix  |  2 ++
 2 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/flake.lock b/flake.lock
index bc114032..ef05c258 100644
--- a/flake.lock
+++ b/flake.lock
@@ -763,11 +763,11 @@
         "treefmt-nix": "treefmt-nix"
       },
       "locked": {
-        "lastModified": 1723343306,
-        "narHash": "sha256-/6sRkPq7/5weX2y0V8sQ29Sz35nt8kyj+BsFtkhgbJE=",
+        "lastModified": 1723512448,
+        "narHash": "sha256-VSTtxGKre1p6zd6ACuBmfDcR+BT9+ml8Y3KrSbfGFYU=",
         "owner": "nix-community",
         "repo": "poetry2nix",
-        "rev": "4a1c112ff0c67f496573dc345bd0b2247818fc29",
+        "rev": "ed52f844c4dd04dde45550c3189529854384124e",
         "type": "github"
       },
       "original": {
@@ -897,11 +897,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1723429325,
-        "narHash": "sha256-4x/32xTCd+xCwFoI/kKSiCr5LQA2ZlyTRYXKEni5HR8=",
+        "lastModified": 1723515680,
+        "narHash": "sha256-nHdKymsHCVIh0Wdm4MvSgxcTTg34FJIYHRQkQYaSuvk=",
         "owner": "oxalica",
         "repo": "rust-overlay",
-        "rev": "65e3dc0fe079fe8df087cd38f1fe6836a0373aad",
+        "rev": "4ee3d9e9569f70d7bb40f28804d6fe950c81eab3",
         "type": "github"
       },
       "original": {
@@ -1035,11 +1035,11 @@
         "nixpkgs": "nixpkgs_7"
       },
       "locked": {
-        "lastModified": 1723450799,
-        "narHash": "sha256-cuT/ce7R2D5Lx6Ted4YS4y+WrAAOXQFAbzLsM1vtPo8=",
+        "lastModified": 1723532088,
+        "narHash": "sha256-6h/P/BkFDw8txlikonKXp5IbluHSPhHJTQRftJLkbLQ=",
         "owner": "danieldk",
         "repo": "tgi-nix",
-        "rev": "29f9b45bd613eced65c4a5241a00aa9346f63d90",
+        "rev": "32335a37ce0f703bab901baf7b74eb11e9972d5f",
         "type": "github"
       },
       "original": {
diff --git a/flake.nix b/flake.nix
index 40b845b7..8784be37 100644
--- a/flake.nix
+++ b/flake.nix
@@ -66,6 +66,7 @@
                 venvShellHook
                 pip
 
+                causal-conv1d
                 click
                 einops
                 fbgemm-gpu
@@ -79,6 +80,7 @@
                 grpcio-tools
                 hf-transfer
                 loguru
+                mamba-ssm
                 marlin-kernels
                 opentelemetry-api
                 opentelemetry-exporter-otlp

From 59922f9bc16afee9efcc7ee1c5f9d753ef314ffa Mon Sep 17 00:00:00 2001
From: "Wang, Yi" <yi.a.wang@intel.com>
Date: Tue, 13 Aug 2024 21:33:55 +0800
Subject: [PATCH 42/72] add numa to improve cpu inference perf (#2330)

Signed-off-by: Wang, Yi A <yi.a.wang@intel.com>
---
 Dockerfile_intel                              | 12 +++----
 .../models/flash_causal_lm.py                 | 31 +++++++++++++++++++
 2 files changed, 35 insertions(+), 8 deletions(-)

diff --git a/Dockerfile_intel b/Dockerfile_intel
index 158c5a89..12480c70 100644
--- a/Dockerfile_intel
+++ b/Dockerfile_intel
@@ -106,7 +106,8 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins
     g++ \
     git \
     wget \
-    cmake
+    cmake \
+    libnuma-dev
 
 ENV HUGGINGFACE_HUB_CACHE=/data \
     HF_HUB_ENABLE_HF_TRANSFER=1 \
@@ -135,7 +136,7 @@ RUN conda install -c conda-forge gperftools mkl
 RUN pip install https://download.pytorch.org/whl/nightly/cpu/torch-2.4.0.dev20240612%2Bcpu-cp310-cp310-linux_x86_64.whl
 RUN pip install https://download.pytorch.org/whl/nightly/cpu/torchvision-0.19.0.dev20240612%2Bcpu-cp310-cp310-linux_x86_64.whl
 RUN pip install https://download.pytorch.org/whl/nightly/cpu/torchaudio-2.4.0.dev20240612%2Bcpu-cp310-cp310-linux_x86_64.whl
-RUN pip install triton
+RUN pip install triton numa
 
 WORKDIR /usr/src
 
@@ -147,16 +148,11 @@ RUN cd intel-extension-for-pytorch && git submodule sync && git submodule update
 
 RUN cd torch-ccl && git submodule sync && git submodule update --init --recursive && pip install .
 
-ENV LD_PRELOAD=/opt/conda/lib/libtcmalloc.so:/opt/conda/lib/libiomp5.so
+ENV LD_PRELOAD=/opt/conda/lib/libtcmalloc.so
 ENV CCL_ROOT=/opt/conda/lib/python3.10/site-packages/oneccl_bindings_for_pytorch
 ENV I_MPI_ROOT=/opt/conda/lib/python3.10/site-packages/oneccl_bindings_for_pytorch
 ENV FI_PROVIDER_PATH=/opt/conda/lib/python3.10/site-packages/oneccl_bindings_for_pytorch/opt/mpi/libfabric/lib/prov:/usr/lib64/libfabric
 ENV LD_LIBRARY_PATH=/opt/conda/lib/python3.10/site-packages/oneccl_bindings_for_pytorch/opt/mpi/libfabric/lib:/opt/conda/lib/python3.10/site-packages/oneccl_bindings_for_pytorch/lib
-ENV KMP_BLOCKTIME=1
-ENV KMP_TPAUSE=0
-ENV KMP_FORKJOIN_BARRIER_PATTERN=dist,dist
-ENV KMP_PLAIN_BARRIER_PATTERN=dist,dist
-ENV KMP_REDUCTION_BARRIER_PATTERN=dist,dist
 
 # Install server
 COPY proto proto
diff --git a/server/text_generation_server/models/flash_causal_lm.py b/server/text_generation_server/models/flash_causal_lm.py
index 21b66a68..42d93a12 100644
--- a/server/text_generation_server/models/flash_causal_lm.py
+++ b/server/text_generation_server/models/flash_causal_lm.py
@@ -74,6 +74,36 @@ def get_sliding_windows() -> int:
     return SLIDING_WINDOW
 
 
+def init_cpu_threads_env(rank_id: int, world_size: int):
+    import importlib.util
+
+    if importlib.util.find_spec("numa") is not None:
+        import numa
+        import psutil
+
+        nodes = numa.get_max_node() + 1
+        rank_per_node = math.ceil(world_size / nodes)
+        num_cpus_per_nodes = int(psutil.cpu_count(logical=False) / nodes)
+        node_id = int(rank_id / rank_per_node)
+        rank_offset_per_node = rank_id % rank_per_node
+        if os.getenv("OMP_NUM_THREADS") is None:
+            num_cpus_per_rank = max(int(num_cpus_per_nodes / rank_per_node), 1)
+        else:
+            num_cpus_per_rank = int(os.getenv("OMP_NUM_THREADS"))
+        if len(numa.get_membind()) == nodes:
+            numa.set_membind([node_id])
+        torch.set_num_threads(num_cpus_per_rank)
+        if len(numa.get_affinity(0)) == psutil.cpu_count(logical=True):
+            cpu_start = num_cpus_per_rank * rank_offset_per_node
+            numa.set_affinity(
+                0,
+                list(numa.node_to_cpus(node_id))[
+                    cpu_start : cpu_start + num_cpus_per_rank
+                ],
+            )
+        logger.info(f"affinity={numa.get_affinity(0)}, membind = {numa.get_membind()}")
+
+
 @dataclass
 class FlashCausalLMBatch(Batch):
     batch_id: int
@@ -854,6 +884,7 @@ class FlashCausalLM(Model):
                 device = torch.device("cpu")
                 # Float16 doesn't exist on target.
                 dtype = torch.bfloat16 if dtype is None else dtype
+                init_cpu_threads_env(rank_id=rank, world_size=world_size)
         else:
             raise NotImplementedError(f"{model_class} is only available on GPU")
 

From 1cebccc72b3233f705724be43ea760239d5d2717 Mon Sep 17 00:00:00 2001
From: drbh <david.richard.holtz@gmail.com>
Date: Tue, 13 Aug 2024 10:19:46 -0400
Subject: [PATCH 43/72] fix: adds causal to attention params (#2408)

fix: adds causal to attention params to check when using flash attn v1
---
 server/text_generation_server/layers/attention/cuda.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/server/text_generation_server/layers/attention/cuda.py b/server/text_generation_server/layers/attention/cuda.py
index d039e1e7..8703eb94 100644
--- a/server/text_generation_server/layers/attention/cuda.py
+++ b/server/text_generation_server/layers/attention/cuda.py
@@ -293,6 +293,7 @@ else:
         max_s,
         softmax_scale,
         window_size_left=-1,
+        causal=None,
         softcap=None,
     ):
         if window_size_left != -1:

From c5fff92b48431fcd38a88cee6042eccbbc958c5f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Dani=C3=ABl=20de=20Kok?= <me@danieldk.eu>
Date: Wed, 14 Aug 2024 11:06:28 +0200
Subject: [PATCH 44/72] nix: partial incremental build of the router (#2416)

This is less incremental than crate2nix, but does build all dependencies
separately, so avoids full rebuilds.
---
 _server.nix |  7 ++++++-
 flake.lock  | 22 ++++++++++++++++++++++
 flake.nix   | 37 +++++++++++++++++++++++++++++--------
 router.nix  | 18 ------------------
 4 files changed, 57 insertions(+), 27 deletions(-)
 delete mode 100644 router.nix

diff --git a/_server.nix b/_server.nix
index bc11ba6b..2cb2f887 100644
--- a/_server.nix
+++ b/_server.nix
@@ -1,4 +1,9 @@
-{ mkPoetryApplication, pkg-config, protobuf, openssl }:
+{
+  mkPoetryApplication,
+  pkg-config,
+  protobuf,
+  openssl,
+}:
 
 mkPoetryApplication {
   # name = "text-generation-server";
diff --git a/flake.lock b/flake.lock
index ef05c258..dab582a5 100644
--- a/flake.lock
+++ b/flake.lock
@@ -579,6 +579,27 @@
         "type": "github"
       }
     },
+    "naersk": {
+      "inputs": {
+        "nixpkgs": [
+          "tgi-nix",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1721727458,
+        "narHash": "sha256-r/xppY958gmZ4oTfLiHN0ZGuQ+RSTijDblVgVLFi1mw=",
+        "owner": "nix-community",
+        "repo": "naersk",
+        "rev": "3fb418eaf352498f6b6c30592e3beb63df42ef11",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "naersk",
+        "type": "github"
+      }
+    },
     "nix-github-actions": {
       "inputs": {
         "nixpkgs": [
@@ -880,6 +901,7 @@
       "inputs": {
         "crate2nix": "crate2nix",
         "flake-utils": "flake-utils_6",
+        "naersk": "naersk",
         "nixpkgs": [
           "tgi-nix",
           "nixpkgs"
diff --git a/flake.nix b/flake.nix
index 8784be37..e1f44212 100644
--- a/flake.nix
+++ b/flake.nix
@@ -7,6 +7,10 @@
     tgi-nix.url = "github:danieldk/tgi-nix";
     nixpkgs.follows = "tgi-nix/nixpkgs";
     flake-utils.url = "github:numtide/flake-utils";
+    naersk = {
+      url = "github:nix-community/naersk";
+      inputs.nixpkgs.follows = "tgi-nix/nixpkgs";
+    };
     poetry2nix.url = "github:nix-community/poetry2nix";
     rust-overlay = {
       url = "github:oxalica/rust-overlay";
@@ -17,6 +21,7 @@
     {
       self,
       crate2nix,
+      naersk,
       nixpkgs,
       flake-utils,
       rust-overlay,
@@ -41,13 +46,32 @@
             tgi-nix.overlay
           ];
         };
+        naersk' = pkgs.callPackage naersk { };
+        router =
+          with pkgs;
+          naersk'.buildPackage {
+            name = "router";
+            src = ./.;
+            cargoBuildOptions =
+              x:
+              x
+              ++ [
+                "-p"
+                "text-generation-router-v3"
+              ];
+            nativeBuildInputs = [ pkg-config ];
+            buildInputs = [
+              openssl.dev
+              protobuf
+            ];
+            doCheck = false;
+          };
 
         inherit (poetry2nix.lib.mkPoetry2Nix { inherit pkgs; }) mkPoetryEditablePackage;
-        text-generation-server = mkPoetryEditablePackage {
-            editablePackageSources = ./server;
-        };
+        text-generation-server = mkPoetryEditablePackage { editablePackageSources = ./server; };
       in
       {
+        defaultPackage = router;
         devShells.default =
           with pkgs;
           mkShell {
@@ -92,11 +116,8 @@
                 transformers
                 vllm
 
-                cargoNix.workspaceMembers.text-generation-launcher.build 
-
-                (callPackage ./router.nix {
-                  inherit (rustPlatform) buildRustPackage importCargoLock;
-                })
+                cargoNix.workspaceMembers.text-generation-launcher.build
+                router
               ]);
 
             venvDir = "./.venv";
diff --git a/router.nix b/router.nix
deleted file mode 100644
index eeeac199..00000000
--- a/router.nix
+++ /dev/null
@@ -1,18 +0,0 @@
-{ buildRustPackage, importCargoLock, pkg-config, protobuf, openssl }:
-
-buildRustPackage {
-  name = "text-generation-router";
-
-  src = ./.;
-
-  sourceDir = ./backends/v3;
-
-  cargoLock = {
-    lockFile = ./Cargo.lock;
-  };
-
-  nativeBuildInputs = [ pkg-config ];
-
-  buildInputs = [ openssl.dev protobuf ];
-
-}

From f3b5c6944173779d4c762702d8369e42a9ce0180 Mon Sep 17 00:00:00 2001
From: Nicolas Patry <patry.nicolas@protonmail.com>
Date: Wed, 14 Aug 2024 11:58:08 +0200
Subject: [PATCH 45/72] Upgrading exl2. (#2415)

* Upgrading exl2.

* Fixing the other pathways.

* Fix idefics.
---
 .gitignore                                    |  2 +-
 Dockerfile                                    |  6 +--
 flake.nix                                     |  1 +
 server/Makefile                               |  1 +
 server/Makefile-exllamav2                     | 12 ++++++
 .../layers/gptq/exllamav2.py                  | 43 ++++++++++++-------
 .../models/causal_lm.py                       |  1 +
 .../models/flash_causal_lm.py                 |  1 +
 .../text_generation_server/models/idefics.py  |  1 +
 .../models/idefics_causal_lm.py               |  1 +
 .../models/seq2seq_lm.py                      |  1 +
 server/text_generation_server/server.py       |  6 +--
 12 files changed, 54 insertions(+), 22 deletions(-)
 create mode 100644 server/Makefile-exllamav2

diff --git a/.gitignore b/.gitignore
index bd9d9125..f79d8faa 100644
--- a/.gitignore
+++ b/.gitignore
@@ -9,7 +9,7 @@ backends/client/src/v3/pb
 
 # ROCm auto-generated files
 *.hip
-server/exllamav2_kernels/exllamav2_kernels/hip/
+server/exllamav2
 server/exllama_kernels/exllama_kernels/hip/
 server/exllama_kernels/exllama_kernels/hip_func/
 *_hip.cuh
diff --git a/Dockerfile b/Dockerfile
index 458ff699..74e7d990 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -123,10 +123,10 @@ RUN TORCH_CUDA_ARCH_LIST="8.0;8.6+PTX" python setup.py build
 # Build Transformers exllama kernels
 FROM kernel-builder AS exllamav2-kernels-builder
 WORKDIR /usr/src
-COPY server/exllamav2_kernels/ .
+COPY server/Makefile-exllamav2/ Makefile
 
 # Build specific version of transformers
-RUN TORCH_CUDA_ARCH_LIST="8.0;8.6+PTX" python setup.py build
+RUN TORCH_CUDA_ARCH_LIST="8.0;8.6+PTX" make build-exllamav2
 
 # Build Transformers awq kernels
 FROM kernel-builder AS awq-kernels-builder
@@ -221,7 +221,7 @@ COPY --from=custom-kernels-builder /usr/src/build/lib.linux-x86_64-cpython-310 /
 # Copy build artifacts from exllama kernels builder
 COPY --from=exllama-kernels-builder /usr/src/build/lib.linux-x86_64-cpython-310 /opt/conda/lib/python3.10/site-packages
 # Copy build artifacts from exllamav2 kernels builder
-COPY --from=exllamav2-kernels-builder /usr/src/build/lib.linux-x86_64-cpython-310 /opt/conda/lib/python3.10/site-packages
+COPY --from=exllamav2-kernels-builder /usr/src/exllamav2/build/lib.linux-x86_64-cpython-310 /opt/conda/lib/python3.10/site-packages
 # Copy build artifacts from awq kernels builder
 COPY --from=awq-kernels-builder /usr/src/llm-awq/awq/kernels/build/lib.linux-x86_64-cpython-310 /opt/conda/lib/python3.10/site-packages
 # Copy build artifacts from eetq kernels builder
diff --git a/flake.nix b/flake.nix
index e1f44212..229184d2 100644
--- a/flake.nix
+++ b/flake.nix
@@ -93,6 +93,7 @@
                 causal-conv1d
                 click
                 einops
+                exllamav2
                 fbgemm-gpu
                 flashinfer
                 flash-attn
diff --git a/server/Makefile b/server/Makefile
index 209fc44e..51ea8b32 100644
--- a/server/Makefile
+++ b/server/Makefile
@@ -6,6 +6,7 @@ include Makefile-eetq
 include Makefile-selective-scan
 include Makefile-lorax-punica
 include Makefile-fbgemm
+include Makefile-exllamav2
 
 unit-tests:
 	pytest -s -vv -m "not private" tests
diff --git a/server/Makefile-exllamav2 b/server/Makefile-exllamav2
new file mode 100644
index 00000000..0d4cc385
--- /dev/null
+++ b/server/Makefile-exllamav2
@@ -0,0 +1,12 @@
+exllamav2_commit := v0.1.8
+
+build-exllamav2:
+	git clone https://github.com/turboderp/exllamav2.git exllamav2 && \
+	cd exllamav2 && git fetch && git checkout $(exllamav2_commit)  && \
+	git submodule update --init --recursive && \
+	pip install -r requirements.txt && \
+	CUDA_ARCH_LIST="8.0;9.0a" NVCC_GENCODE="-gencode=arch=compute_80,code=sm_80 -gencode=arch=compute_90a,code=sm_90a" TORCH_CUDA_ARCH_LIST="8.0;9.0a" python setup.py build
+
+install-exllamav2: build-exllamav2
+	cd exllamav2/ &&  \
+	CUDA_ARCH_LIST="8.0;9.0a" NVCC_GENCODE="-gencode=arch=compute_80,code=sm_80 -gencode=arch=compute_90a,code=sm_90a" TORCH_CUDA_ARCH_LIST="8.0;9.0a" python setup.py install
diff --git a/server/text_generation_server/layers/gptq/exllamav2.py b/server/text_generation_server/layers/gptq/exllamav2.py
index dc3b832f..920a6adf 100644
--- a/server/text_generation_server/layers/gptq/exllamav2.py
+++ b/server/text_generation_server/layers/gptq/exllamav2.py
@@ -12,7 +12,10 @@ from text_generation_server.layers.gptq import GPTQWeight
 from text_generation_server.utils.log import log_master
 
 try:
-    from exllamav2_kernels import make_q_matrix, gemm_half_q_half
+    from exllamav2.ext import exllamav2_ext
+
+    make_q_matrix = exllamav2_ext.make_q_matrix
+    gemm_half_q_half = exllamav2_ext.gemm_half_q_half
 except ImportError:
     log_master(logger.warning, "exllamav2_kernels not installed.")
     raise
@@ -70,6 +73,10 @@ def ext_make_q_matrix(
     """
     Create Q matrix
     """
+    # max_dq_size = 512*(1024**2)
+    # max_dq_rows = max_dq_size // out_features[0]
+    max_dq_rows = 0
+
     # EXL2
     if isinstance(w, Exl2Weight):
         extra.q_group_map = make_group_map(w.q_groups, w.q_weight.shape[0])
@@ -83,10 +90,12 @@ def ext_make_q_matrix(
             w.q_scale_max,
             w.q_groups,
             extra.q_group_map,
-            none_tensor,
-            none_tensor,
-            none_tensor,
+            none_tensor,  # zeros
+            none_tensor,  # scales
+            none_tensor,  # g_idx
+            none_tensor,  # bias
             temp_dq,
+            max_dq_rows,
         )
     # GPTQ
     elif isinstance(w, GPTQWeight):
@@ -106,29 +115,33 @@ def ext_make_q_matrix(
                 w.qweight,
                 extra.q_perm,
                 extra.q_invperm,
-                none_tensor,
-                none_tensor,
-                none_tensor,
-                none_tensor,
+                none_tensor,  # q_scale
+                none_tensor,  # q_scale_max
+                none_tensor,  # q_groups
+                none_tensor,  # q_group_map
                 w.qzeros,
                 w.scales,
                 w.g_idx.cpu(),
+                none_tensor,  # bias
                 temp_dq,
+                max_dq_rows,
             )
         # GPTQ without g_idx
         else:
             return make_q_matrix(
                 w.qweight,
-                none_tensor,
-                none_tensor,
-                none_tensor,
-                none_tensor,
-                none_tensor,
-                none_tensor,
+                none_tensor,  # q_perm
+                none_tensor,  # q_invperm
+                none_tensor,  # q_scale
+                none_tensor,  # q_scale_max
+                none_tensor,  # q_groups
+                none_tensor,  # q_group_map
                 w.qzeros,
                 w.scales,
-                none_tensor,
+                none_tensor,  # g_idx
+                none_tensor,  # bias
                 temp_dq,
+                max_dq_rows,
             )
     else:
         RuntimeError("Cannot create handle")
diff --git a/server/text_generation_server/models/causal_lm.py b/server/text_generation_server/models/causal_lm.py
index 212ab7a9..ba168b13 100644
--- a/server/text_generation_server/models/causal_lm.py
+++ b/server/text_generation_server/models/causal_lm.py
@@ -511,6 +511,7 @@ class CausalLM(Model):
         config_class=AutoConfig,
         batch_class=CausalLMBatch,
     ):
+        self.quantize = quantize
         self.batch_class = batch_class
         self.process_group, rank, world_size = initialize_torch_distributed()
         if torch.cuda.is_available():
diff --git a/server/text_generation_server/models/flash_causal_lm.py b/server/text_generation_server/models/flash_causal_lm.py
index 42d93a12..5e2fd20a 100644
--- a/server/text_generation_server/models/flash_causal_lm.py
+++ b/server/text_generation_server/models/flash_causal_lm.py
@@ -872,6 +872,7 @@ class FlashCausalLM(Model):
         head_size: Optional[int] = None,
         skip_special_tokens: bool = True,
     ):
+        self.quantize = quantize
         self.process_group, rank, world_size = initialize_torch_distributed()
         if torch.cuda.is_available():
             device = torch.device(f"cuda:{rank}")
diff --git a/server/text_generation_server/models/idefics.py b/server/text_generation_server/models/idefics.py
index 29929b98..9058cb96 100644
--- a/server/text_generation_server/models/idefics.py
+++ b/server/text_generation_server/models/idefics.py
@@ -33,6 +33,7 @@ class IDEFICSSharded(IdeficsCausalLM):
         dtype: Optional[torch.dtype] = None,
         trust_remote_code: bool = False,
     ):
+        self.quantize = quantize
         self.process_group, rank, world_size = initialize_torch_distributed()
         if torch.cuda.is_available():
             device = torch.device(f"cuda:{rank}")
diff --git a/server/text_generation_server/models/idefics_causal_lm.py b/server/text_generation_server/models/idefics_causal_lm.py
index 8a80ed68..c5480952 100644
--- a/server/text_generation_server/models/idefics_causal_lm.py
+++ b/server/text_generation_server/models/idefics_causal_lm.py
@@ -580,6 +580,7 @@ class IdeficsCausalLM(Model):
         dtype: Optional[torch.dtype] = None,
         trust_remote_code: bool = False,
     ):
+        self.quantize = quantize
         from text_generation_server.models.custom_modeling.idefics_modeling import (
             IdeficsForVisionText2Text,
         )
diff --git a/server/text_generation_server/models/seq2seq_lm.py b/server/text_generation_server/models/seq2seq_lm.py
index 79c001b0..3c92128a 100644
--- a/server/text_generation_server/models/seq2seq_lm.py
+++ b/server/text_generation_server/models/seq2seq_lm.py
@@ -553,6 +553,7 @@ class Seq2SeqLM(Model):
         tokenizer_class=AutoTokenizer,
         aliases=None,
     ):
+        self.quantize = quantize
         self.process_group, rank, world_size = initialize_torch_distributed()
         if torch.cuda.is_available():
             device = torch.device(f"cuda:{rank}")
diff --git a/server/text_generation_server/server.py b/server/text_generation_server/server.py
index b92ab572..22871ec5 100644
--- a/server/text_generation_server/server.py
+++ b/server/text_generation_server/server.py
@@ -50,12 +50,12 @@ class TextGenerationService(generate_pb2_grpc.TextGenerationServiceServicer):
         self,
         model: Model,
         cache: Cache,
-        quantize: Optional[str],
         server_urls: List[str],
     ):
         self.cache = cache
         self.model = model
-        self.quantize = quantize
+        # Quantize is resolved during model loading
+        self.quantize = model.quantize
         self.server_urls = server_urls
         # For some reason, inference_mode does not work well with GLOO which we use on CPU
         if model.device.type == "cuda":
@@ -255,7 +255,7 @@ def serve(
             ],
         )
         generate_pb2_grpc.add_TextGenerationServiceServicer_to_server(
-            TextGenerationService(model, Cache(), quantize, server_urls), server
+            TextGenerationService(model, Cache(), server_urls), server
         )
         SERVICE_NAMES = (
             generate_pb2.DESCRIPTOR.services_by_name["TextGenerationService"].full_name,

From 3f385991b084c59968817506a54398a1accd03b9 Mon Sep 17 00:00:00 2001
From: Funtowicz Morgan <mfuntowicz@users.noreply.github.com>
Date: Wed, 14 Aug 2024 12:02:05 +0200
Subject: [PATCH 46/72] More fixes trtllm (#2342)

* (backend) use parking_lot crate for RwLock fairness

* (docker) let's put rust in the TRTLLM folder when building

* (docker) build ompi with SLURM support

* (launcher) default new server::run parameters to false for now

* (chore) fmt ... why?
---
 Cargo.lock                     | 1 +
 backends/trtllm/Cargo.toml     | 5 +++--
 backends/trtllm/Dockerfile     | 9 +++++----
 backends/trtllm/src/backend.rs | 3 ++-
 backends/trtllm/src/main.rs    | 8 ++++----
 5 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index bb63422e..d298c379 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4028,6 +4028,7 @@ dependencies = [
  "cxx",
  "cxx-build",
  "log",
+ "parking_lot",
  "pkg-config",
  "text-generation-router",
  "thiserror",
diff --git a/backends/trtllm/Cargo.toml b/backends/trtllm/Cargo.toml
index 7079d3d1..43a114ba 100644
--- a/backends/trtllm/Cargo.toml
+++ b/backends/trtllm/Cargo.toml
@@ -8,17 +8,18 @@ homepage.workspace = true
 [dependencies]
 async-trait = "0.1"
 async-stream = "0.3"
+clap = { version = "4.5", features = ["derive"] }
 cxx = "1.0"
+log = { version = "0.4", features = [] }
 text-generation-router = { path = "../../router" }
 tokenizers = { version = "0.19", features = ["hf-hub"] }
 tokio = { version = "1.38", features = ["rt", "rt-multi-thread", "parking_lot", "signal", "sync"] }
 tokio-stream = "0.1.15"
-clap = { version = "4.5", features = ["derive"] }
 thiserror = "1.0.62"
 tracing = "0.1"
 tracing-opentelemetry = "0.24"
 tracing-subscriber = { version = "0.3", features = ["json", "env-filter"] }
-log = { version = "0.4", features = [] }
+parking_lot = "0.12"
 
 [build-dependencies]
 cmake = "0.1"
diff --git a/backends/trtllm/Dockerfile b/backends/trtllm/Dockerfile
index 60ad03f7..5fd2f89f 100644
--- a/backends/trtllm/Dockerfile
+++ b/backends/trtllm/Dockerfile
@@ -3,7 +3,7 @@ ARG OMPI_VERSION="4.1.6"
 
 # Build dependencies resolver stage
 FROM lukemathwalker/cargo-chef:latest AS chef
-WORKDIR /usr/src/text-generation-inference
+WORKDIR /usr/src/text-generation-inference/backends/trtllm
 
 FROM chef AS planner
 COPY . .
@@ -42,7 +42,7 @@ RUN wget "https://download.open-mpi.org/release/open-mpi/v4.1/$OMPI_TARBALL_FILE
     mkdir /usr/src/mpi && \
     tar -xf "/opt/src/$OMPI_TARBALL_FILENAME" -C /usr/src/mpi --strip-components=1 && \
     cd /usr/src/mpi && \
-    ./configure --prefix=/usr/local/mpi --with-cuda=/usr/local/cuda --without-slurm && \
+    ./configure --prefix=/usr/local/mpi --with-cuda=/usr/local/cuda && \
     make -j all && \
     make install && \
     rm -rf "/opt/src/$OMPI_TARBALL_FILENAME"
@@ -66,7 +66,7 @@ ENV PATH="/root/.cargo/bin:$PATH"
 RUN cargo install cargo-chef
 
 # Cache dependencies
-COPY --from=planner /usr/src/text-generation-inference/recipe.json .
+COPY --from=planner /usr/src/text-generation-inference/backends/trtllm/recipe.json .
 RUN cargo chef cook --release --recipe-path recipe.json
 
 # Build actual TGI
@@ -79,7 +79,8 @@ COPY . .
 COPY --from=trt-builder /usr/local/tensorrt /usr/local/tensorrt
 COPY --from=mpi-builder /usr/local/mpi /usr/local/mpi
 RUN mkdir $TGI_INSTALL_PREFIX && mkdir "$TGI_INSTALL_PREFIX/include" && mkdir "$TGI_INSTALL_PREFIX/lib" && \
-    CMAKE_INSTALL_PREFIX=$TGI_INSTALL_PREFIX cargo build --release --bin text-generation-backends-trtllm
+    cd backends/trtllm && \
+    CMAKE_INSTALL_PREFIX=$TGI_INSTALL_PREFIX cargo build --release
 
 FROM nvidia/cuda:12.5.1-cudnn-runtime-ubuntu22.04 AS runtime
 WORKDIR /usr/local/tgi/bin
diff --git a/backends/trtllm/src/backend.rs b/backends/trtllm/src/backend.rs
index b26d06a6..b23aa6c0 100644
--- a/backends/trtllm/src/backend.rs
+++ b/backends/trtllm/src/backend.rs
@@ -12,12 +12,13 @@ use cxx::UniquePtr;
 use log::{error, warn};
 use tokenizers::Tokenizer;
 use tokio::sync::mpsc::{unbounded_channel, UnboundedSender};
-use tokio::sync::RwLock;
 use tokio::time::{sleep, Instant};
 use tokio_stream::wrappers::UnboundedReceiverStream;
 use tokio_stream::{Stream, StreamExt};
 use tracing::{instrument, span, Level};
 
+// use tokio::sync::RwLock;
+use parking_lot::RwLock;
 use text_generation_router::infer::{Backend, GeneratedText, InferError, InferStreamResponse};
 use text_generation_router::validation::ValidationError::UnsupportedModality;
 use text_generation_router::validation::{Chunk, ValidGenerateRequest, ValidationError};
diff --git a/backends/trtllm/src/main.rs b/backends/trtllm/src/main.rs
index 6d6ee146..e0ba46c7 100644
--- a/backends/trtllm/src/main.rs
+++ b/backends/trtllm/src/main.rs
@@ -1,12 +1,10 @@
+use clap::Parser;
 use std::collections::HashMap;
 use std::path::PathBuf;
-
-use clap::Parser;
-use tokenizers::{FromPretrainedParameters, Tokenizer};
-
 use text_generation_backends_trtllm::errors::TensorRtLlmBackendError;
 use text_generation_backends_trtllm::TensorRtLlmBackend;
 use text_generation_router::server;
+use tokenizers::{FromPretrainedParameters, Tokenizer};
 
 /// App Configuration
 #[derive(Parser, Debug)]
@@ -160,6 +158,8 @@ async fn main() -> Result<(), TensorRtLlmBackendError> {
         messages_api_enabled,
         true,
         max_client_batch_size,
+        false,
+        false,
     )
     .await?;
     Ok(())

From 9aaa12e7ac54d859ebfee4b023f66010cef1bca6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Dani=C3=ABl=20de=20Kok?= <me@danieldk.eu>
Date: Thu, 15 Aug 2024 10:21:51 +0200
Subject: [PATCH 47/72] nix: build router incrementally (#2422)

---
 flake.lock | 22 ----------------------
 flake.nix  | 53 +++++++++++++++++++++++++----------------------------
 2 files changed, 25 insertions(+), 50 deletions(-)

diff --git a/flake.lock b/flake.lock
index dab582a5..ef05c258 100644
--- a/flake.lock
+++ b/flake.lock
@@ -579,27 +579,6 @@
         "type": "github"
       }
     },
-    "naersk": {
-      "inputs": {
-        "nixpkgs": [
-          "tgi-nix",
-          "nixpkgs"
-        ]
-      },
-      "locked": {
-        "lastModified": 1721727458,
-        "narHash": "sha256-r/xppY958gmZ4oTfLiHN0ZGuQ+RSTijDblVgVLFi1mw=",
-        "owner": "nix-community",
-        "repo": "naersk",
-        "rev": "3fb418eaf352498f6b6c30592e3beb63df42ef11",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nix-community",
-        "repo": "naersk",
-        "type": "github"
-      }
-    },
     "nix-github-actions": {
       "inputs": {
         "nixpkgs": [
@@ -901,7 +880,6 @@
       "inputs": {
         "crate2nix": "crate2nix",
         "flake-utils": "flake-utils_6",
-        "naersk": "naersk",
         "nixpkgs": [
           "tgi-nix",
           "nixpkgs"
diff --git a/flake.nix b/flake.nix
index 229184d2..168c8649 100644
--- a/flake.nix
+++ b/flake.nix
@@ -7,10 +7,6 @@
     tgi-nix.url = "github:danieldk/tgi-nix";
     nixpkgs.follows = "tgi-nix/nixpkgs";
     flake-utils.url = "github:numtide/flake-utils";
-    naersk = {
-      url = "github:nix-community/naersk";
-      inputs.nixpkgs.follows = "tgi-nix/nixpkgs";
-    };
     poetry2nix.url = "github:nix-community/poetry2nix";
     rust-overlay = {
       url = "github:oxalica/rust-overlay";
@@ -21,7 +17,6 @@
     {
       self,
       crate2nix,
-      naersk,
       nixpkgs,
       flake-utils,
       rust-overlay,
@@ -34,6 +29,7 @@
         cargoNix = crate2nix.tools.${system}.appliedCargoNix {
           name = "tgi";
           src = ./.;
+          additionalCargoNixArgs = [ "--all-features" ];
         };
         config = {
           allowUnfree = true;
@@ -46,32 +42,10 @@
             tgi-nix.overlay
           ];
         };
-        naersk' = pkgs.callPackage naersk { };
-        router =
-          with pkgs;
-          naersk'.buildPackage {
-            name = "router";
-            src = ./.;
-            cargoBuildOptions =
-              x:
-              x
-              ++ [
-                "-p"
-                "text-generation-router-v3"
-              ];
-            nativeBuildInputs = [ pkg-config ];
-            buildInputs = [
-              openssl.dev
-              protobuf
-            ];
-            doCheck = false;
-          };
-
         inherit (poetry2nix.lib.mkPoetry2Nix { inherit pkgs; }) mkPoetryEditablePackage;
         text-generation-server = mkPoetryEditablePackage { editablePackageSources = ./server; };
       in
       {
-        defaultPackage = router;
         devShells.default =
           with pkgs;
           mkShell {
@@ -118,7 +92,30 @@
                 vllm
 
                 cargoNix.workspaceMembers.text-generation-launcher.build
-                router
+
+                (cargoNix.workspaceMembers.text-generation-router-v3.build.override {
+                  crateOverrides = defaultCrateOverrides // {
+                    aws-lc-rs = attrs: {
+                      # aws-lc-rs does its own custom parsing of Cargo environment
+                      # variables like DEP_.*_INCLUDE. However buildRustCrate does
+                      # not use the version number, so the parsing fails.
+                      postPatch = ''
+                        substituteInPlace build.rs \
+                          --replace-fail \
+                          "assert!(!selected.is_empty()" \
+                          "// assert!(!selected.is_empty()"
+                      '';
+                    };
+                    rav1e = attrs: { env.CARGO_ENCODED_RUSTFLAGS = "-C target-feature=-crt-static"; };
+                    text-generation-router-v3 = attrs: {
+                      # We need to do the src/source root dance so that the build
+                      # has access to the protobuf file.
+                      src = ./.;
+                      postPatch = "cd backends/v3";
+                      buildInputs = [ protobuf ];
+                    };
+                  };
+                })
               ]);
 
             venvDir = "./.venv";

From 57b34958235ee64b7c310a4b5410bcc491a0ef28 Mon Sep 17 00:00:00 2001
From: Nicolas Patry <patry.nicolas@protonmail.com>
Date: Thu, 15 Aug 2024 11:12:51 +0200
Subject: [PATCH 48/72] Fixing exl2 and other quanize tests again. (#2419)

* Fixing exl2 and other quanize tests again.

* Mark exl2 as non release (so CI tests them, needs to be removed latet).

* Fixing exl2 (by disabling cuda graphs)

* Fix quantization defaults without cuda graphs on exl2 (linked to new
issues with it).

* Removing serde override.

* Go back to released exl2 and remove log.

* Adding warnings for deprecated bitsandbytes + upgrade info to warn.
---
 Dockerfile                                    |   3 +
 .../models/test_flash_llama_exl2.py           |   3 -
 launcher/src/main.rs                          | 156 ++++++++++--------
 server/poetry.lock                            |  69 +++++++-
 server/pyproject.toml                         |   1 +
 server/requirements_cuda.txt                  |   4 +
 server/requirements_intel.txt                 |   4 +
 server/requirements_rocm.txt                  |   4 +
 .../models/causal_lm.py                       |   1 +
 server/text_generation_server/models/mamba.py |   1 +
 .../models/seq2seq_lm.py                      |   1 +
 11 files changed, 176 insertions(+), 71 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 74e7d990..b2d274d7 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -250,6 +250,9 @@ RUN cd server && \
     pip install nvidia-nccl-cu12==2.22.3
 
 ENV LD_PRELOAD=/opt/conda/lib/python3.10/site-packages/nvidia/nccl/lib/libnccl.so.2
+# This is needed because exl2 tries to load flash-attn
+# And fails with our builds.
+ENV EXLLAMA_NO_FLASH_ATTN=1
 
 # Deps before the binaries
 # The binaries change on every build given we burn the SHA into them
diff --git a/integration-tests/models/test_flash_llama_exl2.py b/integration-tests/models/test_flash_llama_exl2.py
index 7169c999..18319f60 100644
--- a/integration-tests/models/test_flash_llama_exl2.py
+++ b/integration-tests/models/test_flash_llama_exl2.py
@@ -21,7 +21,6 @@ async def flash_llama_exl2(flash_llama_exl2_handle):
     return flash_llama_exl2_handle.client
 
 
-@pytest.mark.release
 @pytest.mark.asyncio
 @pytest.mark.private
 async def test_flash_llama_exl2(flash_llama_exl2, ignore_logprob_response_snapshot):
@@ -33,7 +32,6 @@ async def test_flash_llama_exl2(flash_llama_exl2, ignore_logprob_response_snapsh
     assert response == ignore_logprob_response_snapshot
 
 
-@pytest.mark.release
 @pytest.mark.asyncio
 @pytest.mark.private
 async def test_flash_llama_exl2_all_params(
@@ -60,7 +58,6 @@ async def test_flash_llama_exl2_all_params(
     assert response == ignore_logprob_response_snapshot
 
 
-@pytest.mark.release
 @pytest.mark.asyncio
 @pytest.mark.private
 async def test_flash_llama_exl2_load(
diff --git a/launcher/src/main.rs b/launcher/src/main.rs
index a64b1d71..9a90a673 100644
--- a/launcher/src/main.rs
+++ b/launcher/src/main.rs
@@ -30,11 +30,18 @@ struct RawConfig {
     n_positions: Option<usize>,
     model_type: Option<String>,
     max_seq_len: Option<usize>,
+    quantization_config: Option<QuantizationConfig>,
+}
+
+#[derive(Deserialize)]
+struct QuantizationConfig {
+    quant_method: Option<Quantization>,
 }
 
 #[derive(Deserialize)]
 struct Config {
     max_position_embeddings: Option<usize>,
+    quantize: Option<Quantization>,
 }
 
 impl From<RawConfig> for Config {
@@ -43,13 +50,16 @@ impl From<RawConfig> for Config {
             .max_position_embeddings
             .or(other.max_seq_len)
             .or(other.n_positions);
+        let quantize = other.quantization_config.and_then(|q| q.quant_method);
         Config {
             max_position_embeddings,
+            quantize,
         }
     }
 }
 
-#[derive(Clone, Copy, Debug, ValueEnum)]
+#[derive(Clone, Copy, Debug, ValueEnum, Deserialize)]
+#[serde(rename_all = "kebab-case")]
 enum Quantization {
     /// 4 bit quantization. Requires a specific AWQ quantized model:
     ///   <https://hf.co/models?search=awq>.
@@ -72,17 +82,17 @@ enum Quantization {
     Marlin,
     /// Bitsandbytes 8bit. Can be applied on any model, will cut the memory requirement in half,
     /// but it is known that the model will be much slower to run than the native f16.
-    #[deprecated(
-        since = "1.1.0",
-        note = "Use `eetq` instead, which provides better latencies overall and is drop-in in most cases"
-    )]
+    // #[deprecated(
+    //     since = "1.1.0",
+    //     note = "Use `eetq` instead, which provides better latencies overall and is drop-in in most cases"
+    // )]
     Bitsandbytes,
     /// Bitsandbytes 4bit. Can be applied on any model, will cut the memory requirement by 4x,
     /// but it is known that the model will be much slower to run than the native f16.
-    BitsandbytesNF4,
+    BitsandbytesNf4,
     /// Bitsandbytes 4bit. nf4 should be preferred in most cases but maybe this one has better
     /// perplexity performance for you model
-    BitsandbytesFP4,
+    BitsandbytesFp4,
     /// [FP8](https://developer.nvidia.com/blog/nvidia-arm-and-intel-publish-fp8-specification-for-standardization-as-an-interchange-format-for-ai/) (e4m3) works on H100 and above
     /// This dtype has native ops should be the fastest if available.
     /// This is currently not the fastest because of local unpacking + padding to satisfy matrix
@@ -99,10 +109,10 @@ impl std::fmt::Display for Quantization {
             Quantization::Bitsandbytes => {
                 write!(f, "bitsandbytes")
             }
-            Quantization::BitsandbytesNF4 => {
+            Quantization::BitsandbytesNf4 => {
                 write!(f, "bitsandbytes-nf4")
             }
-            Quantization::BitsandbytesFP4 => {
+            Quantization::BitsandbytesFp4 => {
                 write!(f, "bitsandbytes-fp4")
             }
             Quantization::Exl2 => {
@@ -1085,6 +1095,7 @@ fn spawn_shards(
     cuda_graphs: Vec<usize>,
     max_total_tokens: usize,
     max_input_tokens: usize,
+    quantize: Option<Quantization>,
     max_log_level: LevelFilter,
     shutdown: Arc<AtomicBool>,
     shutdown_receiver: &mpsc::Receiver<()>,
@@ -1106,7 +1117,6 @@ fn spawn_shards(
         let shutdown_sender = shutdown_sender.clone();
         let otlp_endpoint = args.otlp_endpoint.clone();
         let otlp_service_name = args.otlp_service_name.clone();
-        let quantize = args.quantize;
         let speculate = args.speculate;
         let dtype = args.dtype;
         let trust_remote_code = args.trust_remote_code;
@@ -1429,65 +1439,68 @@ fn main() -> Result<(), LauncherError> {
 
     tracing::info!("{:#?}", args);
 
-    let get_max_position_embeddings = || -> Result<usize, Box<dyn std::error::Error>> {
-        let model_id = args.model_id.clone();
-        let mut path = std::path::Path::new(&args.model_id).to_path_buf();
-        let filename = if !path.exists() {
-            // Assume it's a hub id
+    let get_max_positions_quantize =
+        || -> Result<(usize, Option<Quantization>), Box<dyn std::error::Error>> {
+            let model_id = args.model_id.clone();
+            let mut path = std::path::Path::new(&args.model_id).to_path_buf();
+            let filename = if !path.exists() {
+                // Assume it's a hub id
 
-            let api = if let Ok(token) = std::env::var("HF_TOKEN") {
-                // env variable has precedence over on file token.
-                ApiBuilder::new().with_token(Some(token)).build()?
+                let api = if let Ok(token) = std::env::var("HF_TOKEN") {
+                    // env variable has precedence over on file token.
+                    ApiBuilder::new().with_token(Some(token)).build()?
+                } else {
+                    Api::new()?
+                };
+                let repo = if let Some(ref revision) = args.revision {
+                    api.repo(Repo::with_revision(
+                        model_id,
+                        RepoType::Model,
+                        revision.to_string(),
+                    ))
+                } else {
+                    api.model(model_id)
+                };
+                repo.get("config.json")?
             } else {
-                Api::new()?
+                path.push("config.json");
+                path
             };
-            let repo = if let Some(ref revision) = args.revision {
-                api.repo(Repo::with_revision(
-                    model_id,
-                    RepoType::Model,
-                    revision.to_string(),
-                ))
-            } else {
-                api.model(model_id)
-            };
-            repo.get("config.json")?
-        } else {
-            path.push("config.json");
-            path
-        };
 
-        let content = std::fs::read_to_string(filename)?;
-        let config: RawConfig = serde_json::from_str(&content)?;
+            let content = std::fs::read_to_string(filename)?;
+            let config: RawConfig = serde_json::from_str(&content)?;
 
-        if config.model_type == Some("gemma2".to_string()) {
-            tracing::info!("Forcing flash decoding because of softcap usage");
-            std::env::set_var("ATTENTION", "flashdecoding");
-        }
-        let config: Config = config.into();
-
-        // Quantization usually means you're even more RAM constrained.
-        let max_default = 4096;
-
-        if let Some(max_position_embeddings) = config.max_position_embeddings {
-            if max_position_embeddings > max_default {
-                let max = max_position_embeddings;
-                if args.max_input_tokens.is_none()
-                    && args.max_total_tokens.is_none()
-                    && args.max_batch_prefill_tokens.is_none()
-                {
-                    tracing::info!("Model supports up to {max} but tgi will now set its default to {max_default} instead. This is to save VRAM by refusing large prompts in order to allow more users on the same hardware. You can increase that size using `--max-batch-prefill-tokens={} --max-total-tokens={max} --max-input-tokens={}`.", max + 50, max - 1);
-                }
-                Ok(max_default)
-            } else {
-                Ok(max_position_embeddings)
+            if config.model_type == Some("gemma2".to_string()) {
+                tracing::info!("Forcing flash decoding because of softcap usage");
+                std::env::set_var("ATTENTION", "flashdecoding");
             }
-        } else {
-            Err(Box::new(LauncherError::ArgumentValidation(
-                "no max defined".to_string(),
-            )))
-        }
-    };
-    let max_position_embeddings: usize = get_max_position_embeddings().unwrap_or(4096);
+            let config: Config = config.into();
+            let quantize = config.quantize;
+
+            // Quantization usually means you're even more RAM constrained.
+            let max_default = 4096;
+
+            if let Some(max_position_embeddings) = config.max_position_embeddings {
+                if max_position_embeddings > max_default {
+                    let max = max_position_embeddings;
+                    if args.max_input_tokens.is_none()
+                        && args.max_total_tokens.is_none()
+                        && args.max_batch_prefill_tokens.is_none()
+                    {
+                        tracing::info!("Model supports up to {max} but tgi will now set its default to {max_default} instead. This is to save VRAM by refusing large prompts in order to allow more users on the same hardware. You can increase that size using `--max-batch-prefill-tokens={} --max-total-tokens={max} --max-input-tokens={}`.", max + 50, max - 1);
+                    }
+                    Ok((max_default, quantize))
+                } else {
+                    Ok((max_position_embeddings, quantize))
+                }
+            } else {
+                Err(Box::new(LauncherError::ArgumentValidation(
+                    "no max defined".to_string(),
+                )))
+            }
+        };
+    let (max_position_embeddings, quantize): (usize, Option<Quantization>) =
+        get_max_positions_quantize().unwrap_or((4096, None));
 
     let max_input_tokens = {
         match (args.max_input_tokens, args.max_input_length) {
@@ -1544,18 +1557,26 @@ fn main() -> Result<(), LauncherError> {
         )));
     }
 
-    let cuda_graphs = match (&args.cuda_graphs, &args.quantize) {
+    if matches!(args.quantize, Some(Quantization::Bitsandbytes)) {
+        tracing::warn!("Bitsandbytes is deprecated, use `eetq` instead, which provides better latencies overall and is drop-in in most cases.");
+    }
+    let quantize = args.quantize.or(quantize);
+    let cuda_graphs = match (&args.cuda_graphs, &quantize) {
         (Some(cuda_graphs), _) => cuda_graphs.iter().cloned().filter(|&c| c > 0).collect(),
         #[allow(deprecated)]
         (
             None,
             Some(
                 Quantization::Bitsandbytes
-                | Quantization::BitsandbytesNF4
-                | Quantization::BitsandbytesFP4,
+                | Quantization::BitsandbytesNf4
+                | Quantization::BitsandbytesFp4,
             ),
         ) => {
-            tracing::info!("Bitsandbytes doesn't work with cuda graphs, deactivating them");
+            tracing::warn!("Bitsandbytes doesn't work with cuda graphs, deactivating them");
+            vec![]
+        }
+        (None, Some(Quantization::Exl2)) => {
+            tracing::warn!("Exl2 doesn't work with cuda graphs, deactivating them");
             vec![]
         }
         _ => {
@@ -1672,6 +1693,7 @@ fn main() -> Result<(), LauncherError> {
         cuda_graphs,
         max_total_tokens,
         max_input_tokens,
+        quantize,
         max_log_level,
         shutdown.clone(),
         &shutdown_receiver,
diff --git a/server/poetry.lock b/server/poetry.lock
index 5072aa0b..fc1a54a3 100644
--- a/server/poetry.lock
+++ b/server/poetry.lock
@@ -1070,6 +1070,30 @@ win32-setctime = {version = ">=1.0.0", markers = "sys_platform == \"win32\""}
 [package.extras]
 dev = ["Sphinx (>=4.1.1)", "black (>=19.10b0)", "colorama (>=0.3.4)", "docutils (==0.16)", "flake8 (>=3.7.7)", "isort (>=5.1.1)", "pytest (>=4.6.2)", "pytest-cov (>=2.7.1)", "sphinx-autobuild (>=0.7.1)", "sphinx-rtd-theme (>=0.4.3)", "tox (>=3.9.0)"]
 
+[[package]]
+name = "markdown-it-py"
+version = "3.0.0"
+description = "Python port of markdown-it. Markdown parsing, done right!"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "markdown-it-py-3.0.0.tar.gz", hash = "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb"},
+    {file = "markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1"},
+]
+
+[package.dependencies]
+mdurl = ">=0.1,<1.0"
+
+[package.extras]
+benchmarking = ["psutil", "pytest", "pytest-benchmark"]
+code-style = ["pre-commit (>=3.0,<4.0)"]
+compare = ["commonmark (>=0.9,<1.0)", "markdown (>=3.4,<4.0)", "mistletoe (>=1.0,<2.0)", "mistune (>=2.0,<3.0)", "panflute (>=2.3,<3.0)"]
+linkify = ["linkify-it-py (>=1,<3)"]
+plugins = ["mdit-py-plugins"]
+profiling = ["gprof2dot"]
+rtd = ["jupyter_sphinx", "mdit-py-plugins", "myst-parser", "pyyaml", "sphinx", "sphinx-copybutton", "sphinx-design", "sphinx_book_theme"]
+testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"]
+
 [[package]]
 name = "markupsafe"
 version = "2.1.5"
@@ -1207,6 +1231,17 @@ torch = "*"
 type = "url"
 url = "https://github.com/danieldk/marlin-kernels/releases/download/v0.2.0/marlin_kernels-0.2.0+cu123torch2.4-cp39-cp39-linux_x86_64.whl"
 
+[[package]]
+name = "mdurl"
+version = "0.1.2"
+description = "Markdown URL utilities"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"},
+    {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"},
+]
+
 [[package]]
 name = "mpmath"
 version = "1.3.0"
@@ -2277,6 +2312,20 @@ files = [
 [package.dependencies]
 typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0"
 
+[[package]]
+name = "pygments"
+version = "2.18.0"
+description = "Pygments is a syntax highlighting package written in Python."
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "pygments-2.18.0-py3-none-any.whl", hash = "sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a"},
+    {file = "pygments-2.18.0.tar.gz", hash = "sha256:786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199"},
+]
+
+[package.extras]
+windows-terminal = ["colorama (>=0.4.6)"]
+
 [[package]]
 name = "pytest"
 version = "7.4.4"
@@ -2508,6 +2557,24 @@ urllib3 = ">=1.21.1,<3"
 socks = ["PySocks (>=1.5.6,!=1.5.7)"]
 use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"]
 
+[[package]]
+name = "rich"
+version = "13.7.1"
+description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal"
+optional = false
+python-versions = ">=3.7.0"
+files = [
+    {file = "rich-13.7.1-py3-none-any.whl", hash = "sha256:4edbae314f59eb482f54e9e30bf00d33350aaa94f4bfcd4e9e3110e64d0d7222"},
+    {file = "rich-13.7.1.tar.gz", hash = "sha256:9be308cb1fe2f1f57d67ce99e95af38a1e2bc71ad9813b0e247cf7ffbcc3a432"},
+]
+
+[package.dependencies]
+markdown-it-py = ">=2.2.0"
+pygments = ">=2.13.0,<3.0.0"
+
+[package.extras]
+jupyter = ["ipywidgets (>=7.5.1,<9)"]
+
 [[package]]
 name = "rpds-py"
 version = "0.19.0"
@@ -3584,4 +3651,4 @@ torch = ["torch"]
 [metadata]
 lock-version = "2.0"
 python-versions = ">=3.9,<3.13"
-content-hash = "a89867b23017d2efa8a7aa14d4764bcbd3b4dea9bfbf06a7a68464cb184ac6a1"
+content-hash = "0ff7a244a409b616490cb238995bbe28dedf67ccb8855edafa2b71ee2e777dbd"
diff --git a/server/pyproject.toml b/server/pyproject.toml
index 15da4a8f..57deb1b8 100644
--- a/server/pyproject.toml
+++ b/server/pyproject.toml
@@ -46,6 +46,7 @@ marlin-kernels = [
   { url = "https://github.com/danieldk/marlin-kernels/releases/download/v0.2.0/marlin_kernels-0.2.0+cu123torch2.4-cp311-cp311-linux_x86_64.whl", python = "~3.11", optional = true },
   { url = "https://github.com/danieldk/marlin-kernels/releases/download/v0.2.0/marlin_kernels-0.2.0+cu123torch2.4-cp312-cp312-linux_x86_64.whl", python = "~3.12", optional = true },
 ]
+rich = "^13.7.1"
 
 [tool.poetry.extras]
 torch = ["torch"]
diff --git a/server/requirements_cuda.txt b/server/requirements_cuda.txt
index 828b6fca..eb521bd6 100644
--- a/server/requirements_cuda.txt
+++ b/server/requirements_cuda.txt
@@ -16,6 +16,8 @@ huggingface-hub==0.23.5 ; python_version >= "3.9" and python_version < "3.13"
 idna==3.7 ; python_version >= "3.9" and python_version < "3.13"
 importlib-metadata==7.1.0 ; python_version >= "3.9" and python_version < "3.13"
 loguru==0.6.0 ; python_version >= "3.9" and python_version < "3.13"
+markdown-it-py==3.0.0 ; python_version >= "3.9" and python_version < "3.13"
+mdurl==0.1.2 ; python_version >= "3.9" and python_version < "3.13"
 numpy==1.26.4 ; python_version >= "3.9" and python_version < "3.13"
 opentelemetry-api==1.25.0 ; python_version >= "3.9" and python_version < "3.13"
 opentelemetry-exporter-otlp-proto-common==1.25.0 ; python_version >= "3.9" and python_version < "3.13"
@@ -32,9 +34,11 @@ pillow==10.4.0 ; python_version >= "3.9" and python_version < "3.13"
 prometheus-client==0.20.0 ; python_version >= "3.9" and python_version < "3.13"
 protobuf==4.25.3 ; python_version >= "3.9" and python_version < "3.13"
 py-cpuinfo==9.0.0 ; python_version >= "3.9" and python_version < "3.13"
+pygments==2.18.0 ; python_version >= "3.9" and python_version < "3.13"
 pyyaml==6.0.1 ; python_version >= "3.9" and python_version < "3.13"
 regex==2024.5.15 ; python_version >= "3.9" and python_version < "3.13"
 requests==2.32.3 ; python_version >= "3.9" and python_version < "3.13"
+rich==13.7.1 ; python_version >= "3.9" and python_version < "3.13"
 safetensors==0.4.3 ; python_version >= "3.9" and python_version < "3.13"
 scipy==1.13.1 ; python_version >= "3.9" and python_version < "3.13"
 sentencepiece==0.1.99 ; python_version >= "3.9" and python_version < "3.13"
diff --git a/server/requirements_intel.txt b/server/requirements_intel.txt
index 828b6fca..eb521bd6 100644
--- a/server/requirements_intel.txt
+++ b/server/requirements_intel.txt
@@ -16,6 +16,8 @@ huggingface-hub==0.23.5 ; python_version >= "3.9" and python_version < "3.13"
 idna==3.7 ; python_version >= "3.9" and python_version < "3.13"
 importlib-metadata==7.1.0 ; python_version >= "3.9" and python_version < "3.13"
 loguru==0.6.0 ; python_version >= "3.9" and python_version < "3.13"
+markdown-it-py==3.0.0 ; python_version >= "3.9" and python_version < "3.13"
+mdurl==0.1.2 ; python_version >= "3.9" and python_version < "3.13"
 numpy==1.26.4 ; python_version >= "3.9" and python_version < "3.13"
 opentelemetry-api==1.25.0 ; python_version >= "3.9" and python_version < "3.13"
 opentelemetry-exporter-otlp-proto-common==1.25.0 ; python_version >= "3.9" and python_version < "3.13"
@@ -32,9 +34,11 @@ pillow==10.4.0 ; python_version >= "3.9" and python_version < "3.13"
 prometheus-client==0.20.0 ; python_version >= "3.9" and python_version < "3.13"
 protobuf==4.25.3 ; python_version >= "3.9" and python_version < "3.13"
 py-cpuinfo==9.0.0 ; python_version >= "3.9" and python_version < "3.13"
+pygments==2.18.0 ; python_version >= "3.9" and python_version < "3.13"
 pyyaml==6.0.1 ; python_version >= "3.9" and python_version < "3.13"
 regex==2024.5.15 ; python_version >= "3.9" and python_version < "3.13"
 requests==2.32.3 ; python_version >= "3.9" and python_version < "3.13"
+rich==13.7.1 ; python_version >= "3.9" and python_version < "3.13"
 safetensors==0.4.3 ; python_version >= "3.9" and python_version < "3.13"
 scipy==1.13.1 ; python_version >= "3.9" and python_version < "3.13"
 sentencepiece==0.1.99 ; python_version >= "3.9" and python_version < "3.13"
diff --git a/server/requirements_rocm.txt b/server/requirements_rocm.txt
index 828b6fca..eb521bd6 100644
--- a/server/requirements_rocm.txt
+++ b/server/requirements_rocm.txt
@@ -16,6 +16,8 @@ huggingface-hub==0.23.5 ; python_version >= "3.9" and python_version < "3.13"
 idna==3.7 ; python_version >= "3.9" and python_version < "3.13"
 importlib-metadata==7.1.0 ; python_version >= "3.9" and python_version < "3.13"
 loguru==0.6.0 ; python_version >= "3.9" and python_version < "3.13"
+markdown-it-py==3.0.0 ; python_version >= "3.9" and python_version < "3.13"
+mdurl==0.1.2 ; python_version >= "3.9" and python_version < "3.13"
 numpy==1.26.4 ; python_version >= "3.9" and python_version < "3.13"
 opentelemetry-api==1.25.0 ; python_version >= "3.9" and python_version < "3.13"
 opentelemetry-exporter-otlp-proto-common==1.25.0 ; python_version >= "3.9" and python_version < "3.13"
@@ -32,9 +34,11 @@ pillow==10.4.0 ; python_version >= "3.9" and python_version < "3.13"
 prometheus-client==0.20.0 ; python_version >= "3.9" and python_version < "3.13"
 protobuf==4.25.3 ; python_version >= "3.9" and python_version < "3.13"
 py-cpuinfo==9.0.0 ; python_version >= "3.9" and python_version < "3.13"
+pygments==2.18.0 ; python_version >= "3.9" and python_version < "3.13"
 pyyaml==6.0.1 ; python_version >= "3.9" and python_version < "3.13"
 regex==2024.5.15 ; python_version >= "3.9" and python_version < "3.13"
 requests==2.32.3 ; python_version >= "3.9" and python_version < "3.13"
+rich==13.7.1 ; python_version >= "3.9" and python_version < "3.13"
 safetensors==0.4.3 ; python_version >= "3.9" and python_version < "3.13"
 scipy==1.13.1 ; python_version >= "3.9" and python_version < "3.13"
 sentencepiece==0.1.99 ; python_version >= "3.9" and python_version < "3.13"
diff --git a/server/text_generation_server/models/causal_lm.py b/server/text_generation_server/models/causal_lm.py
index ba168b13..28534d0f 100644
--- a/server/text_generation_server/models/causal_lm.py
+++ b/server/text_generation_server/models/causal_lm.py
@@ -652,6 +652,7 @@ class CausalLM(Model):
             dtype=dtype,
             device=device,
         )
+        self.quantize = quantize
         return self
 
     @property
diff --git a/server/text_generation_server/models/mamba.py b/server/text_generation_server/models/mamba.py
index 5d6ce3c7..f6dcde68 100644
--- a/server/text_generation_server/models/mamba.py
+++ b/server/text_generation_server/models/mamba.py
@@ -412,6 +412,7 @@ class Mamba(Model):
         dtype: Optional[torch.dtype] = None,
         trust_remote_code: bool = False,
     ):
+        self.quantize = quantize
         self.process_group, _rank, world_size = initialize_torch_distributed()
         if world_size > 1:
             raise RuntimeError("Mamba does not support Tensor Parallelism (TP)")
diff --git a/server/text_generation_server/models/seq2seq_lm.py b/server/text_generation_server/models/seq2seq_lm.py
index 3c92128a..04d4c28b 100644
--- a/server/text_generation_server/models/seq2seq_lm.py
+++ b/server/text_generation_server/models/seq2seq_lm.py
@@ -676,6 +676,7 @@ class Seq2SeqLM(Model):
             dtype=dtype,
             device=device,
         )
+        self.quantize = quantize
         return self
 
     @property

From 1b0aa06204d0bad0c06e6352473ccef3aeed7ebf Mon Sep 17 00:00:00 2001
From: Nicolas Patry <patry.nicolas@protonmail.com>
Date: Thu, 15 Aug 2024 13:28:42 +0200
Subject: [PATCH 49/72] Upgrading the tests to match the current workings.
 (#2423)

---
 .../test_bloom_560m/test_bloom_560m.json      |  30 +-
 .../test_flash_deepseek_v2_all_params.json    |   8 +-
 .../test_flash_gemma/test_flash_gemma.json    |  32 +-
 .../test_flash_starcoder_gptq.json            | 146 +-------
 ...t_flash_starcoder_gptq_default_params.json | 144 +-------
 .../test_flash_starcoder_gptq_load.json       | 344 ++++--------------
 .../test_mamba/test_mamba_all_params.json     |  24 +-
 .../test_mt0_base_all_params.json             |   6 +-
 .../models/test_flash_starcoder_gptq.py       |   4 +-
 integration-tests/models/test_mamba.py        |   2 +-
 10 files changed, 166 insertions(+), 574 deletions(-)

diff --git a/integration-tests/models/__snapshots__/test_bloom_560m/test_bloom_560m.json b/integration-tests/models/__snapshots__/test_bloom_560m/test_bloom_560m.json
index b274992e..5d0eeef6 100644
--- a/integration-tests/models/__snapshots__/test_bloom_560m/test_bloom_560m.json
+++ b/integration-tests/models/__snapshots__/test_bloom_560m/test_bloom_560m.json
@@ -11,52 +11,52 @@
       },
       {
         "id": 49833,
-        "logprob": -10.5703125,
+        "logprob": -10.546875,
         "text": " dég"
       },
       {
         "id": 21543,
-        "logprob": -0.14746094,
+        "logprob": -0.18457031,
         "text": "uster"
       },
       {
         "id": 447,
-        "logprob": -1.9277344,
+        "logprob": -1.9287109,
         "text": " un"
       },
       {
         "id": 46341,
-        "logprob": -15.421875,
+        "logprob": -15.4296875,
         "text": " ort"
       },
       {
         "id": 35567,
-        "logprob": -7.5820312,
+        "logprob": -7.578125,
         "text": "olan"
       },
       {
         "id": 15,
-        "logprob": -1.4013672,
+        "logprob": -1.4003906,
         "text": ","
       },
       {
         "id": 1669,
-        "logprob": -1.5664062,
+        "logprob": -1.5439453,
         "text": " il"
       },
       {
         "id": 11580,
-        "logprob": -0.94189453,
+        "logprob": -0.93896484,
         "text": " faut"
       },
       {
         "id": 3913,
-        "logprob": -3.6816406,
+        "logprob": -3.7207031,
         "text": " tout"
       },
       {
         "id": 39261,
-        "logprob": -1.7753906,
+        "logprob": -1.5742188,
         "text": " d'abord"
       }
     ],
@@ -64,13 +64,13 @@
     "tokens": [
       {
         "id": 578,
-        "logprob": -1.6318359,
+        "logprob": -1.6474609,
         "special": false,
         "text": " le"
       },
       {
         "id": 5608,
-        "logprob": -2.4882812,
+        "logprob": -2.4707031,
         "special": false,
         "text": " faire"
       },
@@ -88,19 +88,19 @@
       },
       {
         "id": 693,
-        "logprob": -2.4472656,
+        "logprob": -2.4628906,
         "special": false,
         "text": " à"
       },
       {
         "id": 366,
-        "logprob": -1.1972656,
+        "logprob": -1.1953125,
         "special": false,
         "text": " la"
       },
       {
         "id": 48844,
-        "logprob": -1.7890625,
+        "logprob": -1.7978516,
         "special": false,
         "text": " cass"
       },
diff --git a/integration-tests/models/__snapshots__/test_flash_deepseek_v2/test_flash_deepseek_v2_all_params.json b/integration-tests/models/__snapshots__/test_flash_deepseek_v2/test_flash_deepseek_v2_all_params.json
index 6b45cf6b..3ac8d050 100644
--- a/integration-tests/models/__snapshots__/test_flash_deepseek_v2/test_flash_deepseek_v2_all_params.json
+++ b/integration-tests/models/__snapshots__/test_flash_deepseek_v2/test_flash_deepseek_v2_all_params.json
@@ -11,7 +11,7 @@
       },
       {
         "id": 3533,
-        "logprob": -9.625,
+        "logprob": -9.5625,
         "text": "Test"
       },
       {
@@ -24,13 +24,13 @@
     "tokens": [
       {
         "id": 2143,
-        "logprob": -1.828125,
+        "logprob": -1.8203125,
         "special": false,
         "text": " sent"
       },
       {
         "id": 10081,
-        "logprob": -0.41210938,
+        "logprob": -0.55078125,
         "special": false,
         "text": " successfully"
       },
@@ -42,7 +42,7 @@
       },
       {
         "id": 100001,
-        "logprob": -0.16015625,
+        "logprob": -0.12695312,
         "special": true,
         "text": "<｜end▁of▁sentence｜>"
       }
diff --git a/integration-tests/models/__snapshots__/test_flash_gemma/test_flash_gemma.json b/integration-tests/models/__snapshots__/test_flash_gemma/test_flash_gemma.json
index 8829f9fe..96f2ce17 100644
--- a/integration-tests/models/__snapshots__/test_flash_gemma/test_flash_gemma.json
+++ b/integration-tests/models/__snapshots__/test_flash_gemma/test_flash_gemma.json
@@ -24,13 +24,13 @@
     "tokens": [
       {
         "id": 1736,
-        "logprob": -2.03125,
+        "logprob": -2.046875,
         "special": false,
         "text": " form"
       },
       {
         "id": 109,
-        "logprob": -1.8671875,
+        "logprob": -1.8828125,
         "special": false,
         "text": "\n\n"
       },
@@ -42,48 +42,48 @@
       },
       {
         "id": 2121,
-        "logprob": -1.8125,
+        "logprob": -1.78125,
         "special": false,
         "text": " test"
       },
       {
         "id": 3853,
-        "logprob": -0.24121094,
+        "logprob": -0.23632812,
         "special": false,
         "text": " request"
       },
       {
         "id": 1736,
-        "logprob": -0.100097656,
+        "logprob": -0.09326172,
         "special": false,
         "text": " form"
       },
       {
         "id": 603,
-        "logprob": -0.9453125,
+        "logprob": -0.8828125,
         "special": false,
         "text": " is"
       },
       {
-        "id": 476,
-        "logprob": -1.703125,
+        "id": 1671,
+        "logprob": -1.6171875,
         "special": false,
-        "text": " a"
+        "text": " used"
       },
       {
-        "id": 4551,
-        "logprob": -2.453125,
+        "id": 577,
+        "logprob": -0.390625,
         "special": false,
-        "text": " document"
+        "text": " to"
       },
       {
-        "id": 674,
-        "logprob": -0.796875,
+        "id": 3853,
+        "logprob": -1.2265625,
         "special": false,
-        "text": " that"
+        "text": " request"
       }
     ],
     "top_tokens": null
   },
-  "generated_text": " form\n\nThe test request form is a document that"
+  "generated_text": " form\n\nThe test request form is used to request"
 }
diff --git a/integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq.json b/integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq.json
index 5e537bb7..26224118 100644
--- a/integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq.json
+++ b/integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq.json
@@ -1,8 +1,8 @@
 {
   "details": {
     "best_of_sequences": null,
-    "finish_reason": "length",
-    "generated_tokens": 20,
+    "finish_reason": "eos_token",
+    "generated_tokens": 2,
     "prefill": [
       {
         "id": 589,
@@ -11,57 +11,57 @@
       },
       {
         "id": 3226,
-        "logprob": -8.5859375,
+        "logprob": -8.9453125,
         "text": " ge"
       },
       {
         "id": 21017,
-        "logprob": -7.5859375,
+        "logprob": -8.8515625,
         "text": "ometric"
       },
       {
         "id": 81,
-        "logprob": -0.2668457,
+        "logprob": -0.21875,
         "text": "_"
       },
       {
         "id": 6009,
-        "logprob": -1.6416016,
+        "logprob": -1.2773438,
         "text": "mean"
       },
       {
         "id": 26,
-        "logprob": -0.22705078,
+        "logprob": -0.25195312,
         "text": "("
       },
       {
         "id": 62,
-        "logprob": -5.2304688,
+        "logprob": -4.8203125,
         "text": "L"
       },
       {
         "id": 44,
-        "logprob": -3.0976562,
+        "logprob": -3.7734375,
         "text": ":"
       },
       {
         "id": 1682,
-        "logprob": -1.1044922,
+        "logprob": -0.8310547,
         "text": " List"
       },
       {
         "id": 77,
-        "logprob": -0.14294434,
+        "logprob": -0.22766113,
         "text": "["
       },
       {
         "id": 1808,
-        "logprob": -0.32299805,
+        "logprob": -0.46240234,
         "text": "float"
       },
       {
         "id": 10794,
-        "logprob": -2.8164062,
+        "logprob": -3.0234375,
         "text": "]):"
       }
     ],
@@ -69,126 +69,18 @@
     "tokens": [
       {
         "id": 284,
-        "logprob": -0.1282959,
+        "logprob": -0.04626465,
         "special": false,
         "text": "\n   "
       },
       {
-        "id": 1524,
-        "logprob": -0.97998047,
-        "special": false,
-        "text": " \"\"\""
-      },
-      {
-        "id": 284,
-        "logprob": -0.7006836,
-        "special": false,
-        "text": "\n   "
-      },
-      {
-        "id": 14883,
-        "logprob": -2.1933594,
-        "special": false,
-        "text": " Calculate"
-      },
-      {
-        "id": 322,
-        "logprob": -0.2697754,
-        "special": false,
-        "text": " the"
-      },
-      {
-        "id": 3226,
-        "logprob": -0.0836792,
-        "special": false,
-        "text": " ge"
-      },
-      {
-        "id": 21017,
-        "logprob": -0.018737793,
-        "special": false,
-        "text": "ometric"
-      },
-      {
-        "id": 5651,
-        "logprob": -0.028640747,
-        "special": false,
-        "text": " mean"
-      },
-      {
-        "id": 432,
-        "logprob": -0.29467773,
-        "special": false,
-        "text": " of"
-      },
-      {
-        "id": 312,
-        "logprob": -0.31518555,
-        "special": false,
-        "text": " a"
-      },
-      {
-        "id": 1149,
-        "logprob": -0.20605469,
-        "special": false,
-        "text": " list"
-      },
-      {
-        "id": 432,
-        "logprob": -0.23254395,
-        "special": false,
-        "text": " of"
-      },
-      {
-        "id": 7515,
-        "logprob": -0.4489746,
-        "special": false,
-        "text": " numbers"
-      },
-      {
-        "id": 32,
-        "logprob": -0.6044922,
-        "special": false,
-        "text": "."
-      },
-      {
-        "id": 446,
-        "logprob": -0.63964844,
-        "special": false,
-        "text": "\n\n   "
-      },
-      {
-        "id": 499,
-        "logprob": -1.1953125,
-        "special": false,
-        "text": " :"
-      },
-      {
-        "id": 753,
-        "logprob": -0.03515625,
-        "special": false,
-        "text": "param"
-      },
-      {
-        "id": 498,
-        "logprob": -0.06311035,
-        "special": false,
-        "text": " L"
-      },
-      {
-        "id": 44,
-        "logprob": -0.003414154,
-        "special": false,
-        "text": ":"
-      },
-      {
-        "id": 1682,
-        "logprob": -1.3310547,
-        "special": false,
-        "text": " List"
+        "id": 0,
+        "logprob": null,
+        "special": true,
+        "text": "<|endoftext|>"
       }
     ],
     "top_tokens": null
   },
-  "generated_text": "\n    \"\"\"\n    Calculate the geometric mean of a list of numbers.\n\n    :param L: List"
+  "generated_text": "\n   "
 }
diff --git a/integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq_default_params.json b/integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq_default_params.json
index bf0f5146..015912f8 100644
--- a/integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq_default_params.json
+++ b/integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq_default_params.json
@@ -1,8 +1,8 @@
 {
   "details": {
     "best_of_sequences": null,
-    "finish_reason": "length",
-    "generated_tokens": 20,
+    "finish_reason": "eos_token",
+    "generated_tokens": 2,
     "prefill": [
       {
         "id": 589,
@@ -11,57 +11,57 @@
       },
       {
         "id": 3226,
-        "logprob": -8.5859375,
+        "logprob": -8.9453125,
         "text": " ge"
       },
       {
         "id": 21017,
-        "logprob": -7.5898438,
+        "logprob": -8.859375,
         "text": "ometric"
       },
       {
         "id": 81,
-        "logprob": -0.26586914,
+        "logprob": -0.21984863,
         "text": "_"
       },
       {
         "id": 6009,
-        "logprob": -1.6347656,
+        "logprob": -1.2861328,
         "text": "mean"
       },
       {
         "id": 26,
-        "logprob": -0.22705078,
+        "logprob": -0.25219727,
         "text": "("
       },
       {
         "id": 62,
-        "logprob": -5.2382812,
+        "logprob": -4.8007812,
         "text": "L"
       },
       {
         "id": 44,
-        "logprob": -3.0996094,
+        "logprob": -3.7949219,
         "text": ":"
       },
       {
         "id": 1682,
-        "logprob": -1.1025391,
+        "logprob": -0.8046875,
         "text": " List"
       },
       {
         "id": 77,
-        "logprob": -0.14294434,
+        "logprob": -0.22424316,
         "text": "["
       },
       {
         "id": 1808,
-        "logprob": -0.32226562,
+        "logprob": -0.46191406,
         "text": "float"
       },
       {
         "id": 10794,
-        "logprob": -2.8164062,
+        "logprob": -3.0253906,
         "text": "]):"
       }
     ],
@@ -74,121 +74,13 @@
         "text": "\n   "
       },
       {
-        "id": 442,
-        "logprob": -1.3134766,
-        "special": false,
-        "text": " return"
-      },
-      {
-        "id": 11665,
-        "logprob": -0.10021973,
-        "special": false,
-        "text": " reduce"
-      },
-      {
-        "id": 26,
-        "logprob": 0.0,
-        "special": false,
-        "text": "("
-      },
-      {
-        "id": 5962,
-        "logprob": 0.0,
-        "special": false,
-        "text": "lambda"
-      },
-      {
-        "id": 816,
-        "logprob": 0.0,
-        "special": false,
-        "text": " x"
-      },
-      {
-        "id": 30,
-        "logprob": 0.0,
-        "special": false,
-        "text": ","
-      },
-      {
-        "id": 533,
-        "logprob": 0.0,
-        "special": false,
-        "text": " y"
-      },
-      {
-        "id": 44,
-        "logprob": 0.0,
-        "special": false,
-        "text": ":"
-      },
-      {
-        "id": 816,
-        "logprob": 0.0,
-        "special": false,
-        "text": " x"
-      },
-      {
-        "id": 319,
-        "logprob": -0.42871094,
-        "special": false,
-        "text": " *"
-      },
-      {
-        "id": 533,
-        "logprob": 0.0,
-        "special": false,
-        "text": " y"
-      },
-      {
-        "id": 30,
-        "logprob": 0.0,
-        "special": false,
-        "text": ","
-      },
-      {
-        "id": 498,
-        "logprob": 0.0,
-        "special": false,
-        "text": " L"
-      },
-      {
-        "id": 27,
-        "logprob": 0.0,
-        "special": false,
-        "text": ")"
-      },
-      {
-        "id": 1115,
-        "logprob": 0.0,
-        "special": false,
-        "text": " **"
-      },
-      {
-        "id": 308,
-        "logprob": 0.0,
-        "special": false,
-        "text": " ("
-      },
-      {
-        "id": 35,
-        "logprob": 0.0,
-        "special": false,
-        "text": "1"
-      },
-      {
-        "id": 32,
-        "logprob": -0.31323242,
-        "special": false,
-        "text": "."
-      },
-      {
-        "id": 34,
-        "logprob": 0.0,
-        "special": false,
-        "text": "0"
+        "id": 0,
+        "logprob": null,
+        "special": true,
+        "text": "<|endoftext|>"
       }
     ],
     "top_tokens": null
   },
-  "generated_text": "\n    return reduce(lambda x, y: x * y, L) ** (1.0"
+  "generated_text": "\n   "
 }
diff --git a/integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq_load.json b/integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq_load.json
index 46a21ed8..c9b5ab20 100644
--- a/integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq_load.json
+++ b/integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq_load.json
@@ -2,8 +2,8 @@
   {
     "details": {
       "best_of_sequences": null,
-      "finish_reason": "length",
-      "generated_tokens": 10,
+      "finish_reason": "eos_token",
+      "generated_tokens": 2,
       "prefill": [
         {
           "id": 589,
@@ -12,57 +12,57 @@
         },
         {
           "id": 3226,
-          "logprob": -8.5859375,
+          "logprob": -8.9453125,
           "text": " ge"
         },
         {
           "id": 21017,
-          "logprob": -7.5820312,
+          "logprob": -8.859375,
           "text": "ometric"
         },
         {
           "id": 81,
-          "logprob": -0.26708984,
+          "logprob": -0.21826172,
           "text": "_"
         },
         {
           "id": 6009,
-          "logprob": -1.6386719,
+          "logprob": -1.3085938,
           "text": "mean"
         },
         {
           "id": 26,
-          "logprob": -0.22717285,
+          "logprob": -0.2548828,
           "text": "("
         },
         {
           "id": 62,
-          "logprob": -5.234375,
+          "logprob": -4.8007812,
           "text": "L"
         },
         {
           "id": 44,
-          "logprob": -3.1015625,
+          "logprob": -3.7871094,
           "text": ":"
         },
         {
           "id": 1682,
-          "logprob": -1.1083984,
+          "logprob": -0.81152344,
           "text": " List"
         },
         {
           "id": 77,
-          "logprob": -0.14294434,
+          "logprob": -0.22644043,
           "text": "["
         },
         {
           "id": 1808,
-          "logprob": -0.32592773,
+          "logprob": -0.46313477,
           "text": "float"
         },
         {
           "id": 10794,
-          "logprob": -2.8164062,
+          "logprob": -3.0253906,
           "text": "]):"
         }
       ],
@@ -70,74 +70,26 @@
       "tokens": [
         {
           "id": 284,
-          "logprob": -0.12817383,
+          "logprob": -0.046936035,
           "special": false,
           "text": "\n   "
         },
         {
-          "id": 1524,
-          "logprob": -0.9863281,
-          "special": false,
-          "text": " \"\"\""
-        },
-        {
-          "id": 284,
-          "logprob": -0.7011719,
-          "special": false,
-          "text": "\n   "
-        },
-        {
-          "id": 14883,
-          "logprob": -2.2050781,
-          "special": false,
-          "text": " Calculate"
-        },
-        {
-          "id": 322,
-          "logprob": -0.2668457,
-          "special": false,
-          "text": " the"
-        },
-        {
-          "id": 3226,
-          "logprob": -0.08465576,
-          "special": false,
-          "text": " ge"
-        },
-        {
-          "id": 21017,
-          "logprob": -0.019012451,
-          "special": false,
-          "text": "ometric"
-        },
-        {
-          "id": 5651,
-          "logprob": -0.028625488,
-          "special": false,
-          "text": " mean"
-        },
-        {
-          "id": 432,
-          "logprob": -0.29418945,
-          "special": false,
-          "text": " of"
-        },
-        {
-          "id": 312,
-          "logprob": -0.3161621,
-          "special": false,
-          "text": " a"
+          "id": 0,
+          "logprob": null,
+          "special": true,
+          "text": "<|endoftext|>"
         }
       ],
       "top_tokens": null
     },
-    "generated_text": "\n    \"\"\"\n    Calculate the geometric mean of a"
+    "generated_text": "\n   "
   },
   {
     "details": {
       "best_of_sequences": null,
-      "finish_reason": "length",
-      "generated_tokens": 10,
+      "finish_reason": "eos_token",
+      "generated_tokens": 2,
       "prefill": [
         {
           "id": 589,
@@ -146,57 +98,57 @@
         },
         {
           "id": 3226,
-          "logprob": -8.5859375,
+          "logprob": -8.9375,
           "text": " ge"
         },
         {
           "id": 21017,
-          "logprob": -7.59375,
+          "logprob": -8.859375,
           "text": "ometric"
         },
         {
           "id": 81,
-          "logprob": -0.26953125,
+          "logprob": -0.21899414,
           "text": "_"
         },
         {
           "id": 6009,
-          "logprob": -1.640625,
+          "logprob": -1.3105469,
           "text": "mean"
         },
         {
           "id": 26,
-          "logprob": -0.22705078,
+          "logprob": -0.25561523,
           "text": "("
         },
         {
           "id": 62,
-          "logprob": -5.234375,
+          "logprob": -4.8085938,
           "text": "L"
         },
         {
           "id": 44,
-          "logprob": -3.1132812,
+          "logprob": -3.7890625,
           "text": ":"
         },
         {
           "id": 1682,
-          "logprob": -1.1123047,
+          "logprob": -0.80615234,
           "text": " List"
         },
         {
           "id": 77,
-          "logprob": -0.14294434,
+          "logprob": -0.22375488,
           "text": "["
         },
         {
           "id": 1808,
-          "logprob": -0.32299805,
+          "logprob": -0.46801758,
           "text": "float"
         },
         {
           "id": 10794,
-          "logprob": -2.8164062,
+          "logprob": -3.0253906,
           "text": "]):"
         }
       ],
@@ -204,74 +156,26 @@
       "tokens": [
         {
           "id": 284,
-          "logprob": -0.12854004,
+          "logprob": -0.046447754,
           "special": false,
           "text": "\n   "
         },
         {
-          "id": 1524,
-          "logprob": -0.9897461,
-          "special": false,
-          "text": " \"\"\""
-        },
-        {
-          "id": 284,
-          "logprob": -0.69970703,
-          "special": false,
-          "text": "\n   "
-        },
-        {
-          "id": 14883,
-          "logprob": -2.2050781,
-          "special": false,
-          "text": " Calculate"
-        },
-        {
-          "id": 322,
-          "logprob": -0.2668457,
-          "special": false,
-          "text": " the"
-        },
-        {
-          "id": 3226,
-          "logprob": -0.08496094,
-          "special": false,
-          "text": " ge"
-        },
-        {
-          "id": 21017,
-          "logprob": -0.019012451,
-          "special": false,
-          "text": "ometric"
-        },
-        {
-          "id": 5651,
-          "logprob": -0.029037476,
-          "special": false,
-          "text": " mean"
-        },
-        {
-          "id": 432,
-          "logprob": -0.2939453,
-          "special": false,
-          "text": " of"
-        },
-        {
-          "id": 312,
-          "logprob": -0.31591797,
-          "special": false,
-          "text": " a"
+          "id": 0,
+          "logprob": null,
+          "special": true,
+          "text": "<|endoftext|>"
         }
       ],
       "top_tokens": null
     },
-    "generated_text": "\n    \"\"\"\n    Calculate the geometric mean of a"
+    "generated_text": "\n   "
   },
   {
     "details": {
       "best_of_sequences": null,
-      "finish_reason": "length",
-      "generated_tokens": 10,
+      "finish_reason": "eos_token",
+      "generated_tokens": 2,
       "prefill": [
         {
           "id": 589,
@@ -280,57 +184,57 @@
         },
         {
           "id": 3226,
-          "logprob": -8.5859375,
+          "logprob": -8.9453125,
           "text": " ge"
         },
         {
           "id": 21017,
-          "logprob": -7.5859375,
+          "logprob": -8.859375,
           "text": "ometric"
         },
         {
           "id": 81,
-          "logprob": -0.26586914,
+          "logprob": -0.2163086,
           "text": "_"
         },
         {
           "id": 6009,
-          "logprob": -1.6347656,
+          "logprob": -1.2958984,
           "text": "mean"
         },
         {
           "id": 26,
-          "logprob": -0.22766113,
+          "logprob": -0.2529297,
           "text": "("
         },
         {
           "id": 62,
-          "logprob": -5.2265625,
+          "logprob": -4.796875,
           "text": "L"
         },
         {
           "id": 44,
-          "logprob": -3.0976562,
+          "logprob": -3.7910156,
           "text": ":"
         },
         {
           "id": 1682,
-          "logprob": -1.1025391,
+          "logprob": -0.8076172,
           "text": " List"
         },
         {
           "id": 77,
-          "logprob": -0.1427002,
+          "logprob": -0.22375488,
           "text": "["
         },
         {
           "id": 1808,
-          "logprob": -0.32592773,
+          "logprob": -0.46655273,
           "text": "float"
         },
         {
           "id": 10794,
-          "logprob": -2.8164062,
+          "logprob": -3.0234375,
           "text": "]):"
         }
       ],
@@ -338,74 +242,26 @@
       "tokens": [
         {
           "id": 284,
-          "logprob": -0.13012695,
+          "logprob": -0.0463562,
           "special": false,
           "text": "\n   "
         },
         {
-          "id": 1524,
-          "logprob": -0.98046875,
-          "special": false,
-          "text": " \"\"\""
-        },
-        {
-          "id": 284,
-          "logprob": -0.69921875,
-          "special": false,
-          "text": "\n   "
-        },
-        {
-          "id": 14883,
-          "logprob": -2.1992188,
-          "special": false,
-          "text": " Calculate"
-        },
-        {
-          "id": 322,
-          "logprob": -0.2668457,
-          "special": false,
-          "text": " the"
-        },
-        {
-          "id": 3226,
-          "logprob": -0.083496094,
-          "special": false,
-          "text": " ge"
-        },
-        {
-          "id": 21017,
-          "logprob": -0.01902771,
-          "special": false,
-          "text": "ometric"
-        },
-        {
-          "id": 5651,
-          "logprob": -0.029006958,
-          "special": false,
-          "text": " mean"
-        },
-        {
-          "id": 432,
-          "logprob": -0.29248047,
-          "special": false,
-          "text": " of"
-        },
-        {
-          "id": 312,
-          "logprob": -0.3161621,
-          "special": false,
-          "text": " a"
+          "id": 0,
+          "logprob": null,
+          "special": true,
+          "text": "<|endoftext|>"
         }
       ],
       "top_tokens": null
     },
-    "generated_text": "\n    \"\"\"\n    Calculate the geometric mean of a"
+    "generated_text": "\n   "
   },
   {
     "details": {
       "best_of_sequences": null,
-      "finish_reason": "length",
-      "generated_tokens": 10,
+      "finish_reason": "eos_token",
+      "generated_tokens": 2,
       "prefill": [
         {
           "id": 589,
@@ -414,57 +270,57 @@
         },
         {
           "id": 3226,
-          "logprob": -8.5859375,
+          "logprob": -8.9453125,
           "text": " ge"
         },
         {
           "id": 21017,
-          "logprob": -7.5859375,
+          "logprob": -8.859375,
           "text": "ometric"
         },
         {
           "id": 81,
-          "logprob": -0.26904297,
+          "logprob": -0.21862793,
           "text": "_"
         },
         {
           "id": 6009,
-          "logprob": -1.6386719,
+          "logprob": -1.3095703,
           "text": "mean"
         },
         {
           "id": 26,
-          "logprob": -0.22705078,
+          "logprob": -0.25512695,
           "text": "("
         },
         {
           "id": 62,
-          "logprob": -5.234375,
+          "logprob": -4.796875,
           "text": "L"
         },
         {
           "id": 44,
-          "logprob": -3.1132812,
+          "logprob": -3.7890625,
           "text": ":"
         },
         {
           "id": 1682,
-          "logprob": -1.1074219,
+          "logprob": -0.79589844,
           "text": " List"
         },
         {
           "id": 77,
-          "logprob": -0.14477539,
+          "logprob": -0.22692871,
           "text": "["
         },
         {
           "id": 1808,
-          "logprob": -0.3256836,
+          "logprob": -0.46801758,
           "text": "float"
         },
         {
           "id": 10794,
-          "logprob": -2.8027344,
+          "logprob": -3.0097656,
           "text": "]):"
         }
       ],
@@ -472,67 +328,19 @@
       "tokens": [
         {
           "id": 284,
-          "logprob": -0.12915039,
+          "logprob": -0.04638672,
           "special": false,
           "text": "\n   "
         },
         {
-          "id": 1524,
-          "logprob": -0.98535156,
-          "special": false,
-          "text": " \"\"\""
-        },
-        {
-          "id": 284,
-          "logprob": -0.69921875,
-          "special": false,
-          "text": "\n   "
-        },
-        {
-          "id": 14883,
-          "logprob": -2.2011719,
-          "special": false,
-          "text": " Calculate"
-        },
-        {
-          "id": 322,
-          "logprob": -0.26708984,
-          "special": false,
-          "text": " the"
-        },
-        {
-          "id": 3226,
-          "logprob": -0.08502197,
-          "special": false,
-          "text": " ge"
-        },
-        {
-          "id": 21017,
-          "logprob": -0.019012451,
-          "special": false,
-          "text": "ometric"
-        },
-        {
-          "id": 5651,
-          "logprob": -0.028625488,
-          "special": false,
-          "text": " mean"
-        },
-        {
-          "id": 432,
-          "logprob": -0.29589844,
-          "special": false,
-          "text": " of"
-        },
-        {
-          "id": 312,
-          "logprob": -0.31591797,
-          "special": false,
-          "text": " a"
+          "id": 0,
+          "logprob": null,
+          "special": true,
+          "text": "<|endoftext|>"
         }
       ],
       "top_tokens": null
     },
-    "generated_text": "\n    \"\"\"\n    Calculate the geometric mean of a"
+    "generated_text": "\n   "
   }
 ]
diff --git a/integration-tests/models/__snapshots__/test_mamba/test_mamba_all_params.json b/integration-tests/models/__snapshots__/test_mamba/test_mamba_all_params.json
index ef88926c..93724fe4 100644
--- a/integration-tests/models/__snapshots__/test_mamba/test_mamba_all_params.json
+++ b/integration-tests/models/__snapshots__/test_mamba/test_mamba_all_params.json
@@ -11,22 +11,22 @@
       },
       {
         "id": 13,
-        "logprob": -2.734375,
+        "logprob": -2.59375,
         "text": ","
       },
       {
         "id": 8862,
-        "logprob": -3.6875,
+        "logprob": -3.5625,
         "text": " yellow"
       },
       {
         "id": 13,
-        "logprob": -0.40234375,
+        "logprob": -0.44726562,
         "text": ","
       },
       {
         "id": 209,
-        "logprob": -8.25,
+        "logprob": -8.0,
         "text": " "
       }
     ],
@@ -52,7 +52,7 @@
       },
       {
         "id": 9830,
-        "logprob": -2.25,
+        "logprob": -2.03125,
         "special": false,
         "text": " colors"
       },
@@ -64,13 +64,13 @@
       },
       {
         "id": 329,
-        "logprob": -2.171875,
+        "logprob": -2.734375,
         "special": false,
         "text": " A"
       },
       {
         "id": 1180,
-        "logprob": -2.046875,
+        "logprob": -2.0,
         "special": false,
         "text": " number"
       },
@@ -81,19 +81,19 @@
         "text": " of"
       },
       {
-        "id": 1027,
-        "logprob": -1.5546875,
+        "id": 253,
+        "logprob": -0.69140625,
         "special": false,
-        "text": " different"
+        "text": " the"
       },
       {
         "id": 3295,
-        "logprob": -0.97265625,
+        "logprob": -0.8203125,
         "special": false,
         "text": " color"
       }
     ],
     "top_tokens": null
   },
-  "generated_text": "blue, red, yellow, \nand blue colors. A number of different color"
+  "generated_text": "blue, red, yellow, \nand blue colors. A number of the color"
 }
diff --git a/integration-tests/models/__snapshots__/test_mt0_base/test_mt0_base_all_params.json b/integration-tests/models/__snapshots__/test_mt0_base/test_mt0_base_all_params.json
index 5cacf3e9..40ec7e2f 100644
--- a/integration-tests/models/__snapshots__/test_mt0_base/test_mt0_base_all_params.json
+++ b/integration-tests/models/__snapshots__/test_mt0_base/test_mt0_base_all_params.json
@@ -26,13 +26,13 @@
       },
       {
         "id": 259,
-        "logprob": -0.4716797,
+        "logprob": -0.46948242,
         "special": false,
         "text": " "
       },
       {
         "id": 261,
-        "logprob": -0.044677734,
+        "logprob": -0.15307617,
         "special": false,
         "text": ","
       },
@@ -56,7 +56,7 @@
       },
       {
         "id": 35622,
-        "logprob": -1.1630859,
+        "logprob": -1.2998047,
         "special": false,
         "text": " cloud"
       },
diff --git a/integration-tests/models/test_flash_starcoder_gptq.py b/integration-tests/models/test_flash_starcoder_gptq.py
index f1007d6e..6d46e54d 100644
--- a/integration-tests/models/test_flash_starcoder_gptq.py
+++ b/integration-tests/models/test_flash_starcoder_gptq.py
@@ -21,7 +21,7 @@ async def test_flash_starcoder_gptq(flash_starcoder_gptq, generous_response_snap
         max_new_tokens=20,
         decoder_input_details=True,
     )
-    assert response.details.generated_tokens == 20
+    assert response.details.generated_tokens == 2
     assert response == generous_response_snapshot
 
 
@@ -38,7 +38,7 @@ async def test_flash_starcoder_gptq_default_params(
         decoder_input_details=True,
         seed=0,
     )
-    assert response.details.generated_tokens == 20
+    assert response.details.generated_tokens == 2
     assert response == generous_response_snapshot
 
 
diff --git a/integration-tests/models/test_mamba.py b/integration-tests/models/test_mamba.py
index bc946de8..8548970a 100644
--- a/integration-tests/models/test_mamba.py
+++ b/integration-tests/models/test_mamba.py
@@ -47,7 +47,7 @@ async def test_mamba_all_params(fused_kernel_mamba, response_snapshot):
     assert response.details.generated_tokens == 10
     assert (
         response.generated_text
-        == "blue, red, yellow, \nand blue colors. A number of different color"
+        == "blue, red, yellow, \nand blue colors. A number of the color"
     )
     assert response == response_snapshot
 

From 1411bfb9897bafc4d5fbc61d94b7d295db0ba70a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Dani=C3=ABl=20de=20Kok?= <me@danieldk.eu>
Date: Fri, 16 Aug 2024 10:01:01 +0200
Subject: [PATCH 50/72] nix: try to reduce the number of Rust rebuilds (#2424)

Try to reduce the number of router/launcher rebuilds by filtering
sources. In this way, recompiles should only be triggered by changes
in Cargo or Rust files.
---
 flake.lock              | 16 ++++++++++
 flake.nix               | 30 ++++---------------
 nix/crate-overrides.nix | 66 +++++++++++++++++++++++++++++++++++++++++
 3 files changed, 87 insertions(+), 25 deletions(-)
 create mode 100644 nix/crate-overrides.nix

diff --git a/flake.lock b/flake.lock
index ef05c258..7c772377 100644
--- a/flake.lock
+++ b/flake.lock
@@ -579,6 +579,21 @@
         "type": "github"
       }
     },
+    "nix-filter": {
+      "locked": {
+        "lastModified": 1710156097,
+        "narHash": "sha256-1Wvk8UP7PXdf8bCCaEoMnOT1qe5/Duqgj+rL8sRQsSM=",
+        "owner": "numtide",
+        "repo": "nix-filter",
+        "rev": "3342559a24e85fc164b295c3444e8a139924675b",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "nix-filter",
+        "type": "github"
+      }
+    },
     "nix-github-actions": {
       "inputs": {
         "nixpkgs": [
@@ -880,6 +895,7 @@
       "inputs": {
         "crate2nix": "crate2nix",
         "flake-utils": "flake-utils_6",
+        "nix-filter": "nix-filter",
         "nixpkgs": [
           "tgi-nix",
           "nixpkgs"
diff --git a/flake.nix b/flake.nix
index 168c8649..cf05746a 100644
--- a/flake.nix
+++ b/flake.nix
@@ -4,6 +4,7 @@
       url = "github:nix-community/crate2nix";
       inputs.nixpkgs.follows = "tgi-nix/nixpkgs";
     };
+    nix-filter.url = "github:numtide/nix-filter";
     tgi-nix.url = "github:danieldk/tgi-nix";
     nixpkgs.follows = "tgi-nix/nixpkgs";
     flake-utils.url = "github:numtide/flake-utils";
@@ -17,6 +18,7 @@
     {
       self,
       crate2nix,
+      nix-filter,
       nixpkgs,
       flake-utils,
       rust-overlay,
@@ -44,6 +46,7 @@
         };
         inherit (poetry2nix.lib.mkPoetry2Nix { inherit pkgs; }) mkPoetryEditablePackage;
         text-generation-server = mkPoetryEditablePackage { editablePackageSources = ./server; };
+        crateOverrides = import ./nix/crate-overrides.nix { inherit pkgs nix-filter; };
       in
       {
         devShells.default =
@@ -91,31 +94,8 @@
                 transformers
                 vllm
 
-                cargoNix.workspaceMembers.text-generation-launcher.build
-
-                (cargoNix.workspaceMembers.text-generation-router-v3.build.override {
-                  crateOverrides = defaultCrateOverrides // {
-                    aws-lc-rs = attrs: {
-                      # aws-lc-rs does its own custom parsing of Cargo environment
-                      # variables like DEP_.*_INCLUDE. However buildRustCrate does
-                      # not use the version number, so the parsing fails.
-                      postPatch = ''
-                        substituteInPlace build.rs \
-                          --replace-fail \
-                          "assert!(!selected.is_empty()" \
-                          "// assert!(!selected.is_empty()"
-                      '';
-                    };
-                    rav1e = attrs: { env.CARGO_ENCODED_RUSTFLAGS = "-C target-feature=-crt-static"; };
-                    text-generation-router-v3 = attrs: {
-                      # We need to do the src/source root dance so that the build
-                      # has access to the protobuf file.
-                      src = ./.;
-                      postPatch = "cd backends/v3";
-                      buildInputs = [ protobuf ];
-                    };
-                  };
-                })
+                (cargoNix.workspaceMembers.text-generation-launcher.build.override { inherit crateOverrides; })
+                (cargoNix.workspaceMembers.text-generation-router-v3.build.override { inherit crateOverrides; })
               ]);
 
             venvDir = "./.venv";
diff --git a/nix/crate-overrides.nix b/nix/crate-overrides.nix
new file mode 100644
index 00000000..343b3b25
--- /dev/null
+++ b/nix/crate-overrides.nix
@@ -0,0 +1,66 @@
+{ pkgs, nix-filter }:
+
+let
+  filter = nix-filter.lib;
+in
+with pkgs;
+defaultCrateOverrides
+// {
+  aws-lc-rs = attrs: {
+    # aws-lc-rs does its own custom parsing of Cargo environment
+    # variables like DEP_.*_INCLUDE. However buildRustCrate does
+    # not use the version number, so the parsing fails.
+    postPatch = ''
+      substituteInPlace build.rs \
+        --replace-fail \
+        "assert!(!selected.is_empty()" \
+        "// assert!(!selected.is_empty()"
+    '';
+  };
+  rav1e = attrs: { env.CARGO_ENCODED_RUSTFLAGS = "-C target-feature=-crt-static"; };
+
+  grpc-metadata = attrs: {
+    src =
+      filter {
+        root = ../backends/grpc-metadata;
+        include = with filter; [
+          isDirectory
+          (matchExt "rs")
+        ];
+      };
+  };
+  text-generation-launcer = attrs: {
+    src =
+      filter {
+        root = ../launcher;
+        include = with filter; [
+          isDirectory
+          (matchExt "rs")
+        ];
+      };
+  };
+  text-generation-router = attrs: {
+    src =
+      filter {
+        root = ../router;
+        include = with filter; [
+          isDirectory
+          (matchExt "rs")
+        ];
+      };
+  };
+  text-generation-router-v3 = attrs: {
+    # We need to do the src/source root dance so that the build
+    # has access to the protobuf file.
+    src = filter {
+      root = ../.;
+      include = with filter; [
+        isDirectory
+        (and (inDirectory "backends/v3") (matchExt "rs"))
+        (and (inDirectory "proto") (matchExt "proto"))
+      ];
+    };
+    postPatch = "cd backends/v3";
+    buildInputs = [ protobuf ];
+  };
+}

From 99b662f8c27ddaf469df0f035bc0fdcfa4245bbf Mon Sep 17 00:00:00 2001
From: Vaibhav Srivastav <vaibhavs10@gmail.com>
Date: Fri, 16 Aug 2024 12:43:08 +0200
Subject: [PATCH 51/72] Improve the Consuming TGI + Streaming docs. (#2412)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Improve the Consuming TGI docs.

* Fix erronous update to .

* add info about Open AI client.

* More updates.

* Apply suggestions from code review

Co-authored-by: Erik Kaunismäki <erik.kaum@gmail.com>

* Suggestions from Lucain.

* Update Gradio snippet.

* Up.

* Apply suggestions from code review

Co-authored-by: Lucain <lucainp@gmail.com>

* Update docs/source/basic_tutorials/consuming_tgi.md

Co-authored-by: Lucain <lucainp@gmail.com>

* Up.

* Apply suggestions from code review

Co-authored-by: Omar Sanseviero <osanseviero@gmail.com>

* Up.

* Up.

* Doc review from Nico.

* Doc review from Nico. x2

* Last nit

---------

Co-authored-by: Erik Kaunismäki <erik.kaum@gmail.com>
Co-authored-by: Lucain <lucainp@gmail.com>
Co-authored-by: Omar Sanseviero <osanseviero@gmail.com>
---
 docs/openapi.json                             |   2 +-
 docs/source/basic_tutorials/consuming_tgi.md  | 196 +++++++++++-------
 docs/source/basic_tutorials/using_guidance.md |   2 +-
 docs/source/conceptual/streaming.md           |  94 +++++----
 4 files changed, 174 insertions(+), 120 deletions(-)

diff --git a/docs/openapi.json b/docs/openapi.json
index df21e19d..9ddabaa1 100644
--- a/docs/openapi.json
+++ b/docs/openapi.json
@@ -2094,4 +2094,4 @@
       "description": "Hugging Face Text Generation Inference API"
     }
   ]
-}
+}
\ No newline at end of file
diff --git a/docs/source/basic_tutorials/consuming_tgi.md b/docs/source/basic_tutorials/consuming_tgi.md
index 4829ec7c..6e4ec49c 100644
--- a/docs/source/basic_tutorials/consuming_tgi.md
+++ b/docs/source/basic_tutorials/consuming_tgi.md
@@ -1,81 +1,125 @@
 # Consuming Text Generation Inference
 
-There are many ways you can consume Text Generation Inference server in your applications. After launching, you can use the `/generate` route and make a `POST` request to get results from the server. You can also use the `/generate_stream` route if you want TGI to return a stream of tokens. You can make the requests using the tool of your preference, such as curl, Python or TypeScrpt. For a final end-to-end experience, we also open-sourced ChatUI, a chat interface for open-source models.
+There are many ways to consume Text Generation Inference (TGI) server in your applications. After launching the server, you can use the [Messages API](https://huggingface.co/docs/text-generation-inference/en/messages_api) `/v1/chat/completions` route and make a `POST` request to get results from the server. You can also pass `"stream": true` to the call if you want TGI to return a stream of tokens.
+
+For more information on the API, consult the OpenAPI documentation of `text-generation-inference` available [here](https://huggingface.github.io/text-generation-inference).
+
+You can make the requests using any tool of your preference, such as curl, Python, or TypeScript. For an end-to-end experience, we've open-sourced [ChatUI](https://github.com/huggingface/chat-ui), a chat interface for open-access models.
 
 ## curl
 
-After the launch, you can query the model using either the `/generate` or `/generate_stream` routes:
+After a successful server launch, you can query the model using the `v1/chat/completions` route, to get responses that are compliant to the OpenAI Chat Completion spec:
+
+```bash
+curl localhost:8080/v1/chat/completions \
+    -X POST \
+    -d '{
+  "model": "tgi",
+  "messages": [
+    {
+      "role": "system",
+      "content": "You are a helpful assistant."
+    },
+    {
+      "role": "user",
+      "content": "What is deep learning?"
+    }
+  ],
+  "stream": true,
+  "max_tokens": 20
+}' \
+    -H 'Content-Type: application/json'
+```
+
+For non-chat use-cases, you can also use the `/generate` and `/generate_stream` routes.
 
 ```bash
 curl 127.0.0.1:8080/generate \
     -X POST \
-    -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":20}}' \
+    -d '{
+  "inputs":"What is Deep Learning?",
+  "parameters":{
+    "max_new_tokens":20
+  }
+}' \
     -H 'Content-Type: application/json'
 ```
 
+## Python
 
-## Inference Client
+### Inference Client
 
-[`huggingface-hub`](https://huggingface.co/docs/huggingface_hub/main/en/index) is a Python library to interact with the Hugging Face Hub, including its endpoints. It provides a nice high-level class, [`~huggingface_hub.InferenceClient`], which makes it easy to make calls to a TGI endpoint. `InferenceClient` also takes care of parameter validation and provides a simple to-use interface.
-You can simply install `huggingface-hub` package with pip.
+[`huggingface_hub`](https://huggingface.co/docs/huggingface_hub/main/en/index) is a Python library to interact with the Hugging Face Hub, including its endpoints. It provides a high-level class, [`huggingface_hub.InferenceClient`](https://huggingface.co/docs/huggingface_hub/package_reference/inference_client#huggingface_hub.InferenceClient), which makes it easy to make calls to TGI's Messages API. `InferenceClient` also takes care of parameter validation and provides a simple-to-use interface.
+
+Install `huggingface_hub` package via pip.
 
 ```bash
-pip install huggingface-hub
+pip install huggingface_hub
 ```
 
-Once you start the TGI server, instantiate `InferenceClient()` with the URL to the endpoint serving the model. You can then call `text_generation()` to hit the endpoint through Python.
+You can now use `InferenceClient` the exact same way you would use `OpenAI` client in Python
 
 ```python
 from huggingface_hub import InferenceClient
 
-client = InferenceClient(model="http://127.0.0.1:8080")
-client.text_generation(prompt="Write a code for snake game")
+client = InferenceClient(
+    base_url="http://localhost:8080/v1/",
+)
+
+output = client.chat.completions.create(
+    model="tgi",
+    messages=[
+        {"role": "system", "content": "You are a helpful assistant."},
+        {"role": "user", "content": "Count to 10"},
+    ],
+    stream=True,
+    max_tokens=1024,
+)
+
+for chunk in output:
+    print(chunk.choices[0].delta.content)
 ```
 
-You can do streaming with `InferenceClient` by passing `stream=True`. Streaming will return tokens as they are being generated in the server. To use streaming, you can do as follows:
+You can check out more details about OpenAI compatibility [here](https://huggingface.co/docs/huggingface_hub/en/guides/inference#openai-compatibility).
+
+There is also an async version of the client, `AsyncInferenceClient`, based on `asyncio` and `aiohttp`. You can find docs for it [here](https://huggingface.co/docs/huggingface_hub/package_reference/inference_client#huggingface_hub.AsyncInferenceClient)
+
+### OpenAI Client
+
+You can directly use the OpenAI [Python](https://github.com/openai/openai-python) or [JS](https://github.com/openai/openai-node) clients to interact with TGI.
+
+Install the OpenAI Python package via pip. 
+
+```bash
+pip install openai
+```
 
 ```python
-for token in client.text_generation("How do you make cheese?", max_new_tokens=12, stream=True):
-    print(token)
+from openai import OpenAI
+
+# init the client but point it to TGI
+client = OpenAI(
+    base_url="http://localhost:8080/v1/",
+    api_key="-"
+)
+
+chat_completion = client.chat.completions.create(
+    model="tgi",
+    messages=[
+        {"role": "system", "content": "You are a helpful assistant." },
+        {"role": "user", "content": "What is deep learning?"}
+    ],
+    stream=True
+)
+
+# iterate and print stream
+for message in chat_completion:
+    print(message)
 ```
 
-Another parameter you can use with TGI backend is `details`. You can get more details on generation (tokens, probabilities, etc.) by setting `details` to `True`. When it's specified, TGI will return a `TextGenerationResponse` or `TextGenerationStreamResponse` rather than a string or stream.
+## UI
 
-```python
-output = client.text_generation(prompt="Meaning of life is", details=True)
-print(output)
-
-# TextGenerationResponse(generated_text=' a complex concept that is not always clear to the individual. It is a concept that is not always', details=Details(finish_reason=<FinishReason.Length: 'length'>, generated_tokens=20, seed=None, prefill=[], tokens=[Token(id=267, text=' a', logprob=-2.0723474, special=False), Token(id=11235, text=' complex', logprob=-3.1272552, special=False), Token(id=17908, text=' concept', logprob=-1.3632495, special=False),..))
-```
-
-You can see how to stream below.
-
-```python
-output = client.text_generation(prompt="Meaning of life is", stream=True, details=True)
-print(next(iter(output)))
-
-# TextGenerationStreamResponse(token=Token(id=267, text=' a', logprob=-2.0723474, special=False), generated_text=None, details=None)
-```
-
-You can check out the details of the function [here](https://huggingface.co/docs/huggingface_hub/main/en/package_reference/inference_client#huggingface_hub.InferenceClient.text_generation). There is also an async version of the client, `AsyncInferenceClient`, based on `asyncio` and `aiohttp`. You can find docs for it [here](https://huggingface.co/docs/huggingface_hub/package_reference/inference_client#huggingface_hub.AsyncInferenceClient)
-
-
-## ChatUI
-
-ChatUI is an open-source interface built for LLM serving. It offers many customization options, such as web search with SERP API and more. ChatUI can automatically consume the TGI server and even provides an option to switch between different TGI endpoints. You can try it out at [Hugging Chat](https://huggingface.co/chat/), or use the [ChatUI Docker Space](https://huggingface.co/new-space?template=huggingchat/chat-ui-template) to deploy your own Hugging Chat to Spaces.
-
-To serve both ChatUI and TGI in same environment, simply add your own endpoints to the `MODELS` variable in `.env.local` file inside the `chat-ui` repository. Provide the endpoints pointing to where TGI is served.
-
-```
-{
-// rest of the model config here
-"endpoints": [{"url": "https://HOST:PORT/generate_stream"}]
-}
-```
-
-![ChatUI](https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/chatui_screen.png)
-
-## Gradio
+### Gradio
 
 Gradio is a Python library that helps you build web applications for your machine learning models with a few lines of code. It has a `ChatInterface` wrapper that helps create neat UIs for chatbots. Let's take a look at how to create a chatbot with streaming mode using TGI and Gradio. Let's install Gradio and Hub Python library first.
 
@@ -89,19 +133,28 @@ Assume you are serving your model on port 8080, we will query through [Inference
 import gradio as gr
 from huggingface_hub import InferenceClient
 
-client = InferenceClient(model="http://127.0.0.1:8080")
+client = InferenceClient(base_url="http://127.0.0.1:8080")
 
 def inference(message, history):
     partial_message = ""
-    for token in client.text_generation(message, max_new_tokens=20, stream=True):
-        partial_message += token
+    output = client.chat.completions.create(
+        messages=[
+            {"role": "system", "content": "You are a helpful assistant."},
+            {"role": "user", "content": message},
+        ],
+        stream=True,
+        max_tokens=1024,
+    )
+    
+    for chunk in output:
+        partial_message += chunk.choices[0].delta.content
         yield partial_message
 
 gr.ChatInterface(
     inference,
     chatbot=gr.Chatbot(height=300),
     textbox=gr.Textbox(placeholder="Chat with me!", container=False, scale=7),
-    description="This is the demo for Gradio UI consuming TGI endpoint with LLaMA 7B-Chat model.",
+    description="This is the demo for Gradio UI consuming TGI endpoint.",
     title="Gradio 🤝 TGI",
     examples=["Are tomatoes vegetables?"],
     retry_btn="Retry",
@@ -110,20 +163,7 @@ gr.ChatInterface(
 ).queue().launch()
 ```
 
-The UI looks like this 👇
-
-<div class="flex justify-center">
-    <img
-        class="block dark:hidden"
-        src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/tgi/gradio-tgi.png"
-    />
-    <img
-        class="hidden dark:block"
-        src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/tgi/gradio-tgi-dark.png"
-    />
-</div>
-
-You can try the demo directly here 👇
+You can check out the UI and try the demo directly here 👇
 
 <div class="block dark:hidden">
 	<iframe
@@ -141,15 +181,19 @@ You can try the demo directly here 👇
 </div>
 
 
-You can disable streaming mode using `return` instead of `yield` in your inference function, like below.
-
-```python
-def inference(message, history):
-    return client.text_generation(message, max_new_tokens=20)
-```
-
 You can read more about how to customize a `ChatInterface` [here](https://www.gradio.app/guides/creating-a-chatbot-fast).
 
-## API documentation
+### ChatUI
 
-You can consult the OpenAPI documentation of the `text-generation-inference` REST API using the `/docs` route. The Swagger UI is also available [here](https://huggingface.github.io/text-generation-inference).
+[ChatUI](https://github.com/huggingface/chat-ui) is an open-source interface built for consuming LLMs. It offers many customization options, such as web search with SERP API and more. ChatUI can automatically consume the TGI server and even provides an option to switch between different TGI endpoints. You can try it out at [Hugging Chat](https://huggingface.co/chat/), or use the [ChatUI Docker Space](https://huggingface.co/new-space?template=huggingchat/chat-ui-template) to deploy your own Hugging Chat to Spaces.
+
+To serve both ChatUI and TGI in same environment, simply add your own endpoints to the `MODELS` variable in `.env.local` file inside the `chat-ui` repository. Provide the endpoints pointing to where TGI is served.
+
+```
+{
+// rest of the model config here
+"endpoints": [{"url": "https://HOST:PORT/generate_stream"}]
+}
+```
+
+![ChatUI](https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/chatui_screen.png)
\ No newline at end of file
diff --git a/docs/source/basic_tutorials/using_guidance.md b/docs/source/basic_tutorials/using_guidance.md
index d0008fdb..f09ef348 100644
--- a/docs/source/basic_tutorials/using_guidance.md
+++ b/docs/source/basic_tutorials/using_guidance.md
@@ -4,7 +4,7 @@ Text Generation Inference (TGI) now supports [JSON and regex grammars](#grammar-
 
 These feature are available starting from version `1.4.3`. They are accessible via the [`huggingface_hub`](https://pypi.org/project/huggingface-hub/) library. The tool support is compatible with OpenAI's client libraries. The following guide will walk you through the new features and how to use them!
 
-_note: guidance is supported as grammar in the `/generate` endpoint and as tools in the `/chat/completions` endpoint._
+_note: guidance is supported as grammar in the `/generate` endpoint and as tools in the `v1/chat/completions` endpoint._
 
 ## How it works
 
diff --git a/docs/source/conceptual/streaming.md b/docs/source/conceptual/streaming.md
index 71ec9b25..49c48fa0 100644
--- a/docs/source/conceptual/streaming.md
+++ b/docs/source/conceptual/streaming.md
@@ -48,34 +48,29 @@ To stream tokens with `InferenceClient`, simply pass `stream=True` and iterate o
 ```python
 from huggingface_hub import InferenceClient
 
-client = InferenceClient("http://127.0.0.1:8080")
-for token in client.text_generation("How do you make cheese?", max_new_tokens=12, stream=True):
-    print(token)
+client = InferenceClient(base_url="http://127.0.0.1:8080")
+output = client.chat.completions.create(
+    messages=[
+        {"role": "system", "content": "You are a helpful assistant."},
+        {"role": "user", "content": "Count to 10"},
+    ],
+    stream=True,
+    max_tokens=1024,
+)
 
-# To
-# make
-# cheese
-#,
-# you
-# need
-# to
-# start
-# with
-# milk
-#.
-```
+for chunk in output:
+    print(chunk.choices[0].delta.content)
 
-If you want additional details, you can add `details=True`. In this case, you get a `TextGenerationStreamResponse` which contains additional information such as the probabilities and the tokens. For the final response in the stream, it also returns the full generated text.
-
-```python
-for details in client.text_generation("How do you make cheese?", max_new_tokens=12, details=True, stream=True):
-    print(details)
-
-#TextGenerationStreamResponse(token=Token(id=193, text='\n', logprob=-0.007358551, special=False), generated_text=None, details=None)
-#TextGenerationStreamResponse(token=Token(id=2044, text='To', logprob=-1.1357422, special=False), generated_text=None, details=None)
-#TextGenerationStreamResponse(token=Token(id=717, text=' make', logprob=-0.009841919, special=False), generated_text=None, details=None)
-#...
-#TextGenerationStreamResponse(token=Token(id=25, text='.', logprob=-1.3408203, special=False), generated_text='\nTo make cheese, you need to start with milk.', details=StreamDetails(finish_reason=<FinishReason.Length: 'length'>, generated_tokens=12, seed=None))
+# 1
+# 2
+# 3
+# 4
+# 5
+# 6
+# 7
+# 8
+# 9
+# 10
 ```
 
 The `huggingface_hub` library also comes with an `AsyncInferenceClient` in case you need to handle the requests concurrently.
@@ -83,31 +78,46 @@ The `huggingface_hub` library also comes with an `AsyncInferenceClient` in case
 ```python
 from huggingface_hub import AsyncInferenceClient
 
-client = AsyncInferenceClient("http://127.0.0.1:8080")
-async for token in await client.text_generation("How do you make cheese?", stream=True):
-    print(token)
+client = AsyncInferenceClient(base_url="http://127.0.0.1:8080")
+async def main():
+    stream = await client.chat.completions.create(
+        messages=[{"role": "user", "content": "Say this is a test"}],
+        stream=True,
+    )
+    async for chunk in stream:
+        print(chunk.choices[0].delta.content or "", end="")
 
-# To
-# make
-# cheese
-#,
-# you
-# need
-# to
-# start
-# with
-# milk
+asyncio.run(main())
+
+# This
+# is
+# a
+# test
 #.
 ```
 
 ### Streaming with cURL
 
-To use the `generate_stream` endpoint with curl, you can add the `-N` flag, which disables curl default buffering and shows data as it arrives from the server
+To use the OpenAI Chat Completions compatible Messages API `v1/chat/completions` endpoint with curl, you can add the `-N` flag, which disables curl default buffering and shows data as it arrives from the server
 
 ```curl
-curl -N 127.0.0.1:8080/generate_stream \
+curl localhost:8080/v1/chat/completions \
     -X POST \
-    -d '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":20}}' \
+    -d '{
+  "model": "tgi",
+  "messages": [
+    {
+      "role": "system",
+      "content": "You are a helpful assistant."
+    },
+    {
+      "role": "user",
+      "content": "What is deep learning?"
+    }
+  ],
+  "stream": true,
+  "max_tokens": 20
+}' \
     -H 'Content-Type: application/json'
 ```
 

From c7ab1810d4cc1baa25b869ced1adff3ac2c8e3c3 Mon Sep 17 00:00:00 2001
From: Nicolas Patry <patry.nicolas@protonmail.com>
Date: Fri, 16 Aug 2024 13:21:44 +0200
Subject: [PATCH 52/72] Further fixes. (#2426)

* Further fixes.

* Update the conftest to allow NaN (first logprob).

* Fix the condition.
---
 Dockerfile                                    |   4 +-
 integration-tests/conftest.py                 |   1 +
 .../test_flash_llama_fp8_load.json            |  96 ++++----
 .../test_flash_starcoder_gptq_load.json       | 224 +++++++++---------
 .../models/test_flash_llama_fp8.py            |   7 +-
 5 files changed, 168 insertions(+), 164 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index b2d274d7..ed0c4aa3 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -40,14 +40,14 @@ RUN cargo build --profile release-opt
 
 # Python builder
 # Adapted from: https://github.com/pytorch/pytorch/blob/master/Dockerfile
-FROM nvidia/cuda:12.1.0-devel-ubuntu22.04 AS pytorch-install
+FROM nvidia/cuda:12.4.1-devel-ubuntu22.04 AS pytorch-install
 
 # NOTE: When updating PyTorch version, beware to remove `pip install nvidia-nccl-cu12==2.22.3` below in the Dockerfile. Context: https://github.com/huggingface/text-generation-inference/pull/2099
 ARG PYTORCH_VERSION=2.4.0
 
 ARG PYTHON_VERSION=3.10
 # Keep in sync with `server/pyproject.toml
-ARG CUDA_VERSION=12.1
+ARG CUDA_VERSION=12.4
 ARG MAMBA_VERSION=24.3.0-0
 ARG CUDA_CHANNEL=nvidia
 ARG INSTALL_CHANNEL=pytorch
diff --git a/integration-tests/conftest.py b/integration-tests/conftest.py
index 46a8769f..15af1cad 100644
--- a/integration-tests/conftest.py
+++ b/integration-tests/conftest.py
@@ -118,6 +118,7 @@ class ResponseComparator(JSONSnapshotExtension):
                 and token.text == other.text
                 and (
                     self.ignore_logprob
+                    or (token.logprob == other.logprob and token.logprob is None)
                     or math.isclose(token.logprob, other.logprob, rel_tol=self.rtol)
                 )
                 and token.special == other.special
diff --git a/integration-tests/models/__snapshots__/test_flash_llama_fp8/test_flash_llama_fp8_load.json b/integration-tests/models/__snapshots__/test_flash_llama_fp8/test_flash_llama_fp8_load.json
index 36c87c09..1e9ff6c0 100644
--- a/integration-tests/models/__snapshots__/test_flash_llama_fp8/test_flash_llama_fp8_load.json
+++ b/integration-tests/models/__snapshots__/test_flash_llama_fp8/test_flash_llama_fp8_load.json
@@ -12,12 +12,12 @@
         },
         {
           "id": 2323,
-          "logprob": -9.421875,
+          "logprob": -9.5625,
           "text": "Test"
         },
         {
           "id": 1715,
-          "logprob": -10.546875,
+          "logprob": -10.375,
           "text": " request"
         }
       ],
@@ -25,61 +25,61 @@
       "tokens": [
         {
           "id": 369,
-          "logprob": -2.1816406,
+          "logprob": -2.15625,
           "special": false,
           "text": " for"
         },
         {
           "id": 279,
-          "logprob": -2.6992188,
+          "logprob": -2.703125,
           "special": false,
           "text": " the"
         },
         {
           "id": 220,
-          "logprob": -3.6308594,
+          "logprob": -3.640625,
           "special": false,
           "text": " "
         },
         {
           "id": 679,
-          "logprob": -1.7988281,
+          "logprob": -1.703125,
           "special": false,
           "text": "201"
         },
         {
           "id": 24,
-          "logprob": -1.3535156,
+          "logprob": -1.421875,
           "special": false,
           "text": "9"
         },
         {
           "id": 12,
-          "logprob": -2.0058594,
+          "logprob": -2.03125,
           "special": false,
           "text": "-"
         },
         {
           "id": 2366,
-          "logprob": -0.45410156,
+          "logprob": -0.49023438,
           "special": false,
           "text": "202"
         },
         {
           "id": 15,
-          "logprob": -0.037109375,
+          "logprob": -0.041503906,
           "special": false,
           "text": "0"
         },
         {
           "id": 2978,
-          "logprob": -0.8095703,
+          "logprob": -0.87109375,
           "special": false,
           "text": " school"
         },
         {
           "id": 1060,
-          "logprob": -0.013053894,
+          "logprob": -0.012939453,
           "special": false,
           "text": " year"
         }
@@ -101,12 +101,12 @@
         },
         {
           "id": 2323,
-          "logprob": -9.421875,
+          "logprob": -9.5625,
           "text": "Test"
         },
         {
           "id": 1715,
-          "logprob": -10.546875,
+          "logprob": -10.375,
           "text": " request"
         }
       ],
@@ -114,61 +114,61 @@
       "tokens": [
         {
           "id": 369,
-          "logprob": -2.1816406,
+          "logprob": -2.15625,
           "special": false,
           "text": " for"
         },
         {
           "id": 279,
-          "logprob": -2.6992188,
+          "logprob": -2.703125,
           "special": false,
           "text": " the"
         },
         {
           "id": 220,
-          "logprob": -3.6308594,
+          "logprob": -3.640625,
           "special": false,
           "text": " "
         },
         {
           "id": 679,
-          "logprob": -1.7988281,
+          "logprob": -1.703125,
           "special": false,
           "text": "201"
         },
         {
           "id": 24,
-          "logprob": -1.3535156,
+          "logprob": -1.421875,
           "special": false,
           "text": "9"
         },
         {
           "id": 12,
-          "logprob": -2.0058594,
+          "logprob": -2.03125,
           "special": false,
           "text": "-"
         },
         {
           "id": 2366,
-          "logprob": -0.45410156,
+          "logprob": -0.49023438,
           "special": false,
           "text": "202"
         },
         {
           "id": 15,
-          "logprob": -0.037109375,
+          "logprob": -0.041503906,
           "special": false,
           "text": "0"
         },
         {
           "id": 2978,
-          "logprob": -0.8095703,
+          "logprob": -0.87109375,
           "special": false,
           "text": " school"
         },
         {
           "id": 1060,
-          "logprob": -0.013053894,
+          "logprob": -0.012939453,
           "special": false,
           "text": " year"
         }
@@ -190,12 +190,12 @@
         },
         {
           "id": 2323,
-          "logprob": -9.421875,
+          "logprob": -9.5625,
           "text": "Test"
         },
         {
           "id": 1715,
-          "logprob": -10.546875,
+          "logprob": -10.375,
           "text": " request"
         }
       ],
@@ -203,61 +203,61 @@
       "tokens": [
         {
           "id": 369,
-          "logprob": -2.1816406,
+          "logprob": -2.15625,
           "special": false,
           "text": " for"
         },
         {
           "id": 279,
-          "logprob": -2.6992188,
+          "logprob": -2.703125,
           "special": false,
           "text": " the"
         },
         {
           "id": 220,
-          "logprob": -3.6308594,
+          "logprob": -3.640625,
           "special": false,
           "text": " "
         },
         {
           "id": 679,
-          "logprob": -1.7988281,
+          "logprob": -1.703125,
           "special": false,
           "text": "201"
         },
         {
           "id": 24,
-          "logprob": -1.3535156,
+          "logprob": -1.421875,
           "special": false,
           "text": "9"
         },
         {
           "id": 12,
-          "logprob": -2.0058594,
+          "logprob": -2.03125,
           "special": false,
           "text": "-"
         },
         {
           "id": 2366,
-          "logprob": -0.45410156,
+          "logprob": -0.49023438,
           "special": false,
           "text": "202"
         },
         {
           "id": 15,
-          "logprob": -0.037109375,
+          "logprob": -0.041503906,
           "special": false,
           "text": "0"
         },
         {
           "id": 2978,
-          "logprob": -0.8095703,
+          "logprob": -0.87109375,
           "special": false,
           "text": " school"
         },
         {
           "id": 1060,
-          "logprob": -0.013053894,
+          "logprob": -0.012939453,
           "special": false,
           "text": " year"
         }
@@ -279,12 +279,12 @@
         },
         {
           "id": 2323,
-          "logprob": -9.421875,
+          "logprob": -9.5625,
           "text": "Test"
         },
         {
           "id": 1715,
-          "logprob": -10.546875,
+          "logprob": -10.375,
           "text": " request"
         }
       ],
@@ -292,61 +292,61 @@
       "tokens": [
         {
           "id": 369,
-          "logprob": -2.1816406,
+          "logprob": -2.15625,
           "special": false,
           "text": " for"
         },
         {
           "id": 279,
-          "logprob": -2.6992188,
+          "logprob": -2.703125,
           "special": false,
           "text": " the"
         },
         {
           "id": 220,
-          "logprob": -3.6308594,
+          "logprob": -3.640625,
           "special": false,
           "text": " "
         },
         {
           "id": 679,
-          "logprob": -1.7988281,
+          "logprob": -1.703125,
           "special": false,
           "text": "201"
         },
         {
           "id": 24,
-          "logprob": -1.3535156,
+          "logprob": -1.421875,
           "special": false,
           "text": "9"
         },
         {
           "id": 12,
-          "logprob": -2.0058594,
+          "logprob": -2.03125,
           "special": false,
           "text": "-"
         },
         {
           "id": 2366,
-          "logprob": -0.45410156,
+          "logprob": -0.49023438,
           "special": false,
           "text": "202"
         },
         {
           "id": 15,
-          "logprob": -0.037109375,
+          "logprob": -0.041503906,
           "special": false,
           "text": "0"
         },
         {
           "id": 2978,
-          "logprob": -0.8095703,
+          "logprob": -0.87109375,
           "special": false,
           "text": " school"
         },
         {
           "id": 1060,
-          "logprob": -0.013053894,
+          "logprob": -0.012939453,
           "special": false,
           "text": " year"
         }
diff --git a/integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq_load.json b/integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq_load.json
index c9b5ab20..d9072c52 100644
--- a/integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq_load.json
+++ b/integration-tests/models/__snapshots__/test_flash_starcoder_gptq/test_flash_starcoder_gptq_load.json
@@ -17,37 +17,37 @@
         },
         {
           "id": 21017,
-          "logprob": -8.859375,
+          "logprob": -8.8515625,
           "text": "ometric"
         },
         {
           "id": 81,
-          "logprob": -0.21826172,
+          "logprob": -0.22033691,
           "text": "_"
         },
         {
           "id": 6009,
-          "logprob": -1.3085938,
+          "logprob": -1.2939453,
           "text": "mean"
         },
         {
           "id": 26,
-          "logprob": -0.2548828,
+          "logprob": -0.25268555,
           "text": "("
         },
         {
           "id": 62,
-          "logprob": -4.8007812,
+          "logprob": -4.796875,
           "text": "L"
         },
         {
           "id": 44,
-          "logprob": -3.7871094,
+          "logprob": -3.796875,
           "text": ":"
         },
         {
           "id": 1682,
-          "logprob": -0.81152344,
+          "logprob": -0.8066406,
           "text": " List"
         },
         {
@@ -57,7 +57,7 @@
         },
         {
           "id": 1808,
-          "logprob": -0.46313477,
+          "logprob": -0.46166992,
           "text": "float"
         },
         {
@@ -70,7 +70,7 @@
       "tokens": [
         {
           "id": 284,
-          "logprob": -0.046936035,
+          "logprob": -0.046844482,
           "special": false,
           "text": "\n   "
         },
@@ -103,22 +103,22 @@
         },
         {
           "id": 21017,
-          "logprob": -8.859375,
+          "logprob": -8.8515625,
           "text": "ometric"
         },
         {
           "id": 81,
-          "logprob": -0.21899414,
+          "logprob": -0.21826172,
           "text": "_"
         },
         {
           "id": 6009,
-          "logprob": -1.3105469,
+          "logprob": -1.2871094,
           "text": "mean"
         },
         {
           "id": 26,
-          "logprob": -0.25561523,
+          "logprob": -0.25390625,
           "text": "("
         },
         {
@@ -131,92 +131,6 @@
           "logprob": -3.7890625,
           "text": ":"
         },
-        {
-          "id": 1682,
-          "logprob": -0.80615234,
-          "text": " List"
-        },
-        {
-          "id": 77,
-          "logprob": -0.22375488,
-          "text": "["
-        },
-        {
-          "id": 1808,
-          "logprob": -0.46801758,
-          "text": "float"
-        },
-        {
-          "id": 10794,
-          "logprob": -3.0253906,
-          "text": "]):"
-        }
-      ],
-      "seed": null,
-      "tokens": [
-        {
-          "id": 284,
-          "logprob": -0.046447754,
-          "special": false,
-          "text": "\n   "
-        },
-        {
-          "id": 0,
-          "logprob": null,
-          "special": true,
-          "text": "<|endoftext|>"
-        }
-      ],
-      "top_tokens": null
-    },
-    "generated_text": "\n   "
-  },
-  {
-    "details": {
-      "best_of_sequences": null,
-      "finish_reason": "eos_token",
-      "generated_tokens": 2,
-      "prefill": [
-        {
-          "id": 589,
-          "logprob": null,
-          "text": "def"
-        },
-        {
-          "id": 3226,
-          "logprob": -8.9453125,
-          "text": " ge"
-        },
-        {
-          "id": 21017,
-          "logprob": -8.859375,
-          "text": "ometric"
-        },
-        {
-          "id": 81,
-          "logprob": -0.2163086,
-          "text": "_"
-        },
-        {
-          "id": 6009,
-          "logprob": -1.2958984,
-          "text": "mean"
-        },
-        {
-          "id": 26,
-          "logprob": -0.2529297,
-          "text": "("
-        },
-        {
-          "id": 62,
-          "logprob": -4.796875,
-          "text": "L"
-        },
-        {
-          "id": 44,
-          "logprob": -3.7910156,
-          "text": ":"
-        },
         {
           "id": 1682,
           "logprob": -0.8076172,
@@ -224,12 +138,12 @@
         },
         {
           "id": 77,
-          "logprob": -0.22375488,
+          "logprob": -0.22302246,
           "text": "["
         },
         {
           "id": 1808,
-          "logprob": -0.46655273,
+          "logprob": -0.46435547,
           "text": "float"
         },
         {
@@ -242,7 +156,7 @@
       "tokens": [
         {
           "id": 284,
-          "logprob": -0.0463562,
+          "logprob": -0.046722412,
           "special": false,
           "text": "\n   "
         },
@@ -275,47 +189,133 @@
         },
         {
           "id": 21017,
-          "logprob": -8.859375,
+          "logprob": -8.8515625,
           "text": "ometric"
         },
         {
           "id": 81,
-          "logprob": -0.21862793,
+          "logprob": -0.21813965,
           "text": "_"
         },
         {
           "id": 6009,
-          "logprob": -1.3095703,
+          "logprob": -1.2744141,
           "text": "mean"
         },
         {
           "id": 26,
-          "logprob": -0.25512695,
+          "logprob": -0.2512207,
           "text": "("
         },
         {
           "id": 62,
-          "logprob": -4.796875,
+          "logprob": -4.8046875,
           "text": "L"
         },
         {
           "id": 44,
-          "logprob": -3.7890625,
+          "logprob": -3.7851562,
           "text": ":"
         },
         {
           "id": 1682,
-          "logprob": -0.79589844,
+          "logprob": -0.81396484,
           "text": " List"
         },
         {
           "id": 77,
-          "logprob": -0.22692871,
+          "logprob": -0.22570801,
           "text": "["
         },
         {
           "id": 1808,
-          "logprob": -0.46801758,
+          "logprob": -0.46044922,
+          "text": "float"
+        },
+        {
+          "id": 10794,
+          "logprob": -3.0234375,
+          "text": "]):"
+        }
+      ],
+      "seed": null,
+      "tokens": [
+        {
+          "id": 284,
+          "logprob": -0.04650879,
+          "special": false,
+          "text": "\n   "
+        },
+        {
+          "id": 0,
+          "logprob": null,
+          "special": true,
+          "text": "<|endoftext|>"
+        }
+      ],
+      "top_tokens": null
+    },
+    "generated_text": "\n   "
+  },
+  {
+    "details": {
+      "best_of_sequences": null,
+      "finish_reason": "eos_token",
+      "generated_tokens": 2,
+      "prefill": [
+        {
+          "id": 589,
+          "logprob": null,
+          "text": "def"
+        },
+        {
+          "id": 3226,
+          "logprob": -8.9453125,
+          "text": " ge"
+        },
+        {
+          "id": 21017,
+          "logprob": -8.8515625,
+          "text": "ometric"
+        },
+        {
+          "id": 81,
+          "logprob": -0.21960449,
+          "text": "_"
+        },
+        {
+          "id": 6009,
+          "logprob": -1.2890625,
+          "text": "mean"
+        },
+        {
+          "id": 26,
+          "logprob": -0.25073242,
+          "text": "("
+        },
+        {
+          "id": 62,
+          "logprob": -4.8085938,
+          "text": "L"
+        },
+        {
+          "id": 44,
+          "logprob": -3.8046875,
+          "text": ":"
+        },
+        {
+          "id": 1682,
+          "logprob": -0.8071289,
+          "text": " List"
+        },
+        {
+          "id": 77,
+          "logprob": -0.22570801,
+          "text": "["
+        },
+        {
+          "id": 1808,
+          "logprob": -0.46118164,
           "text": "float"
         },
         {
@@ -328,7 +328,7 @@
       "tokens": [
         {
           "id": 284,
-          "logprob": -0.04638672,
+          "logprob": -0.046539307,
           "special": false,
           "text": "\n   "
         },
diff --git a/integration-tests/models/test_flash_llama_fp8.py b/integration-tests/models/test_flash_llama_fp8.py
index fe5df590..1980846d 100644
--- a/integration-tests/models/test_flash_llama_fp8.py
+++ b/integration-tests/models/test_flash_llama_fp8.py
@@ -21,6 +21,7 @@ async def test_flash_llama_fp8(flash_llama_fp8, response_snapshot):
         "Test request", max_new_tokens=10, decoder_input_details=True
     )
 
+    assert response.generated_text == " for the 2019-2020 school year"
     assert response.details.generated_tokens == 10
     assert response == response_snapshot
 
@@ -57,6 +58,8 @@ async def test_flash_llama_fp8_load(flash_llama_fp8, generate_load, response_sna
     )
 
     assert len(responses) == 4
-    assert all([r.generated_text == responses[0].generated_text for r in responses])
-
+    assert responses[0].generated_text == " for the 2019-2020 school year"
+    assert all(
+        [r.generated_text == responses[0].generated_text for r in responses]
+    ), f"Different messages : {[r.generated_text for r in responses]}"
     assert responses == response_snapshot

From cb0a29484d573125336a02dc2191479f18cacabe Mon Sep 17 00:00:00 2001
From: Nicolas Patry <patry.nicolas@protonmail.com>
Date: Fri, 16 Aug 2024 14:21:29 +0200
Subject: [PATCH 53/72] FIxing the CI.

---
 docs/openapi.json                            | 2 +-
 docs/source/basic_tutorials/consuming_tgi.md | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/openapi.json b/docs/openapi.json
index 9ddabaa1..df21e19d 100644
--- a/docs/openapi.json
+++ b/docs/openapi.json
@@ -2094,4 +2094,4 @@
       "description": "Hugging Face Text Generation Inference API"
     }
   ]
-}
\ No newline at end of file
+}
diff --git a/docs/source/basic_tutorials/consuming_tgi.md b/docs/source/basic_tutorials/consuming_tgi.md
index 6e4ec49c..b07e7219 100644
--- a/docs/source/basic_tutorials/consuming_tgi.md
+++ b/docs/source/basic_tutorials/consuming_tgi.md
@@ -88,7 +88,7 @@ There is also an async version of the client, `AsyncInferenceClient`, based on `
 
 You can directly use the OpenAI [Python](https://github.com/openai/openai-python) or [JS](https://github.com/openai/openai-node) clients to interact with TGI.
 
-Install the OpenAI Python package via pip. 
+Install the OpenAI Python package via pip.
 
 ```bash
 pip install openai
@@ -145,7 +145,7 @@ def inference(message, history):
         stream=True,
         max_tokens=1024,
     )
-    
+
     for chunk in output:
         partial_message += chunk.choices[0].delta.content
         yield partial_message
@@ -196,4 +196,4 @@ To serve both ChatUI and TGI in same environment, simply add your own endpoints
 }
 ```
 
-![ChatUI](https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/chatui_screen.png)
\ No newline at end of file
+![ChatUI](https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/chatui_screen.png)

From 53729b74acb89c272ab449bf9e0b55859a8430cb Mon Sep 17 00:00:00 2001
From: Hugo Larcher <hugo.larcher@huggingface.co>
Date: Fri, 16 Aug 2024 19:43:30 +0200
Subject: [PATCH 54/72] doc: Add metrics documentation and add a 'Reference'
 section (#2230)

* doc: Add metrics documentation and add a 'Reference' section

* doc: Add API reference

* doc: Refactor API reference

* fix: Message API link

* Bad rebase

* Moving the docs.

---------

Co-authored-by: Nicolas Patry <patry.nicolas@protonmail.com>
---
 .github/workflows/build_pr_documentation.yaml |   2 +-
 .pre-commit-config.yaml                       |   2 +-
 docs/source/_toctree.yml                      |  14 ++-
 .../api_reference.md}                         |  33 +++--
 .../launcher.md                               |   0
 docs/source/reference/metrics.md              |  30 +++++
 router/src/server.rs                          | 114 ++++++++++++++++++
 update_doc.py                                 |   2 +-
 8 files changed, 179 insertions(+), 18 deletions(-)
 rename docs/source/{messages_api.md => reference/api_reference.md} (84%)
 rename docs/source/{basic_tutorials => reference}/launcher.md (100%)
 create mode 100644 docs/source/reference/metrics.md

diff --git a/.github/workflows/build_pr_documentation.yaml b/.github/workflows/build_pr_documentation.yaml
index bf03bfdf..a5ce39a5 100644
--- a/.github/workflows/build_pr_documentation.yaml
+++ b/.github/workflows/build_pr_documentation.yaml
@@ -11,7 +11,7 @@ concurrency:
 
 jobs:
   build:
-    uses: huggingface/doc-builder/.github/workflows/build_pr_documentation.yaml@main
+    uses: huggingface/doc-builder/.github/workflows/build_pr_documentation.yml@main
     with:
       commit_sha: ${{ github.event.pull_request.head.sha }}
       pr_number: ${{ github.event.number }}
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 6f5e685e..0c8b6885 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -5,7 +5,7 @@ repos:
     -   id: check-yaml
     -   id: end-of-file-fixer
     -   id: trailing-whitespace
-        exclude: docs/source/basic_tutorials/launcher.md
+        exclude: docs/source/reference/launcher.md
 -   repo: https://github.com/psf/black
     rev: 24.2.0
     hooks:
diff --git a/docs/source/_toctree.yml b/docs/source/_toctree.yml
index e97c00aa..f52fa2ec 100644
--- a/docs/source/_toctree.yml
+++ b/docs/source/_toctree.yml
@@ -17,8 +17,6 @@
     title: Installation from source
   - local: supported_models
     title: Supported Models and Hardware
-  - local: messages_api
-    title: Messages API
   - local: architecture
     title: Internal Architecture
   - local: usage_statistics
@@ -33,8 +31,6 @@
     title: Serving Private & Gated Models
   - local: basic_tutorials/using_cli
     title: Using TGI CLI
-  - local: basic_tutorials/launcher
-    title: All TGI CLI options
   - local: basic_tutorials/non_core_models
     title: Non-core Model Serving
   - local: basic_tutorials/safety
@@ -48,6 +44,14 @@
   - local: basic_tutorials/train_medusa
     title: Train Medusa
   title: Tutorials
+- sections:
+  - local: reference/launcher
+    title: All TGI CLI options
+  - local: reference/metrics
+    title: Exported Metrics
+  - local: reference/api_reference
+    title: API Reference
+  title: Reference
 - sections:
   - local: conceptual/streaming
     title: Streaming
@@ -64,7 +68,7 @@
   - local: conceptual/speculation
     title: Speculation (Medusa, ngram)
   - local: conceptual/guidance
-    title: How Guidance Works (via outlines
+    title: How Guidance Works (via outlines)
   - local: conceptual/lora
     title: LoRA (Low-Rank Adaptation)
 
diff --git a/docs/source/messages_api.md b/docs/source/reference/api_reference.md
similarity index 84%
rename from docs/source/messages_api.md
rename to docs/source/reference/api_reference.md
index 250aaae2..52043c80 100644
--- a/docs/source/messages_api.md
+++ b/docs/source/reference/api_reference.md
@@ -1,18 +1,31 @@
-# Messages API
+# HTTP API Reference
+
+#### Table of Contents
+
+- [Text Generation Inference custom API](#text-generation-inference-custom-api)
+- [OpenAI Messages API](#openai-messages-api)
+  - [Making a Request](#making-a-request)
+  - [Streaming](#streaming)
+  - [Synchronous](#synchronous)
+  - [Hugging Face Inference Endpoints](#hugging-face-inference-endpoints)
+  - [Cloud Providers](#cloud-providers)
+      - [Amazon SageMaker](#amazon-sagemaker)
+
+The HTTP API is a RESTful API that allows you to interact with the text-generation-inference component. Two endpoints are available:
+* Text Generation Inference [custom API](https://huggingface.github.io/text-generation-inference/)
+* OpenAI's [Messages API](#openai-messages-api)
+
+
+## Text Generation Inference custom API
+
+Check the [API documentation](https://huggingface.github.io/text-generation-inference/) for more information on how to interact with the Text Generation Inference API.
+
+## OpenAI Messages API
 
 Text Generation Inference (TGI) now supports the Messages API, which is fully compatible with the OpenAI Chat Completion API. This feature is available starting from version 1.4.0. You can use OpenAI's client libraries or third-party libraries expecting OpenAI schema to interact with TGI's Messages API. Below are some examples of how to utilize this compatibility.
 
 > **Note:** The Messages API is supported from TGI version 1.4.0 and above. Ensure you are using a compatible version to access this feature.
 
-#### Table of Contents
-
-- [Making a Request](#making-a-request)
-- [Streaming](#streaming)
-- [Synchronous](#synchronous)
-- [Hugging Face Inference Endpoints](#hugging-face-inference-endpoints)
-- [Cloud Providers](#cloud-providers)
-  - [Amazon SageMaker](#amazon-sagemaker)
-
 ## Making a Request
 
 You can make a request to TGI's Messages API using `curl`. Here's an example:
diff --git a/docs/source/basic_tutorials/launcher.md b/docs/source/reference/launcher.md
similarity index 100%
rename from docs/source/basic_tutorials/launcher.md
rename to docs/source/reference/launcher.md
diff --git a/docs/source/reference/metrics.md b/docs/source/reference/metrics.md
new file mode 100644
index 00000000..d34d38ea
--- /dev/null
+++ b/docs/source/reference/metrics.md
@@ -0,0 +1,30 @@
+# Metrics
+
+TGI exposes multiple metrics that can be collected via the `/metrics` Prometheus endpoint.
+These metrics can be used to monitor the performance of TGI, autoscale deployment and to help identify bottlenecks.
+
+The following metrics are exposed:
+
+| Metric Name                                | Description                                                                              | Type      | Unit    |
+|--------------------------------------------|------------------------------------------------------------------------------------------|-----------|---------|
+| `tgi_batch_current_max_tokens`             | Maximum tokens for the current batch                                                     | Gauge     | Count   |
+| `tgi_batch_current_size`                   | Current batch size                                                                       | Gauge     | Count   |
+| `tgi_batch_decode_duration`                | Time spent decoding a batch per method (prefill or decode)                               | Histogram | Seconds |
+| `tgi_batch_filter_duration`                | Time spent filtering batches and sending generated tokens per method (prefill or decode) | Histogram | Seconds |
+| `tgi_batch_forward_duration`               | Batch forward duration per method (prefill or decode)                                    | Histogram | Seconds |
+| `tgi_batch_inference_count`                | Inference calls per method (prefill or decode)                                           | Counter   | Count   |
+| `tgi_batch_inference_duration`             | Batch inference duration                                                                 | Histogram | Seconds |
+| `tgi_batch_inference_success`              | Number of successful inference calls per method (prefill or decode)                      | Counter   | Count   |
+| `tgi_batch_next_size`                      | Batch size of the next batch                                                             | Histogram | Count   |
+| `tgi_queue_size`                           | Current queue size                                                                       | Gauge     | Count   |
+| `tgi_request_count`                        | Total number of requests                                                                 | Counter   | Count   |
+| `tgi_request_duration`                     | Total time spent processing the request (e2e latency)                                    | Histogram | Seconds |
+| `tgi_request_generated_tokens`             | Generated tokens per request                                                             | Histogram | Count   |
+| `tgi_request_inference_duration`           | Request inference duration                                                               | Histogram | Seconds |
+| `tgi_request_input_length`                 | Input token length per request                                                           | Histogram | Count   |
+| `tgi_request_max_new_tokens`               | Maximum new tokens per request                                                           | Histogram | Count   |
+| `tgi_request_mean_time_per_token_duration` | Mean time per token per request (inter-token latency)                                    | Histogram | Seconds |
+| `tgi_request_queue_duration`               | Time spent in the queue per request                                                      | Histogram | Seconds |
+| `tgi_request_skipped_tokens`               | Speculated tokens per request                                                            | Histogram | Count   |
+| `tgi_request_success`                      | Number of successful requests                                                            | Counter   |         |
+| `tgi_request_validation_duration`          | Time spent validating the request                                                        | Histogram | Seconds |
diff --git a/router/src/server.rs b/router/src/server.rs
index ab268efa..8ec7a871 100644
--- a/router/src/server.rs
+++ b/router/src/server.rs
@@ -2003,6 +2003,120 @@ async fn start(
         .install_recorder()
         .expect("failed to install metrics recorder");
 
+    // Metrics descriptions
+    metrics::describe_counter!("tgi_request_success", "Number of successful requests");
+    metrics::describe_histogram!(
+        "tgi_request_duration",
+        metrics::Unit::Seconds,
+        "Request duration"
+    );
+    metrics::describe_histogram!(
+        "tgi_request_validation_duration",
+        metrics::Unit::Seconds,
+        "Request validation duration"
+    );
+    metrics::describe_histogram!(
+        "tgi_request_queue_duration",
+        metrics::Unit::Seconds,
+        "Request queue duration"
+    );
+    metrics::describe_histogram!(
+        "tgi_request_inference_duration",
+        metrics::Unit::Seconds,
+        "Request inference duration"
+    );
+    metrics::describe_histogram!(
+        "tgi_request_mean_time_per_token_duration",
+        metrics::Unit::Seconds,
+        "Mean time per token per request"
+    );
+    metrics::describe_histogram!(
+        "tgi_request_generated_tokens",
+        metrics::Unit::Count,
+        "Generated tokens per request"
+    );
+    metrics::describe_counter!(
+        "tgi_batch_inference_count",
+        metrics::Unit::Count,
+        "Inference calls per method (prefill or decode)"
+    );
+    metrics::describe_counter!(
+        "tgi_request_count",
+        metrics::Unit::Count,
+        "Total number of requests"
+    );
+    metrics::describe_counter!(
+        "tgi_batch_inference_success",
+        metrics::Unit::Count,
+        "Number of successful inference calls per method (prefill or decode)"
+    );
+    metrics::describe_gauge!(
+        "tgi_batch_current_size",
+        metrics::Unit::Count,
+        "Current batch size"
+    );
+    metrics::describe_gauge!("tgi_queue_size", metrics::Unit::Count, "Current queue size");
+    metrics::describe_gauge!(
+        "tgi_batch_current_max_tokens",
+        metrics::Unit::Count,
+        "Maximum tokens for the current batch"
+    );
+    metrics::describe_histogram!(
+        "tgi_request_max_new_tokens",
+        metrics::Unit::Count,
+        "Maximum new tokens per request"
+    );
+    metrics::describe_histogram!(
+        "tgi_batch_inference_duration",
+        metrics::Unit::Seconds,
+        "Batch inference duration"
+    );
+    metrics::describe_histogram!(
+        "tgi_batch_forward_duration",
+        metrics::Unit::Seconds,
+        "Batch forward duration per method (prefill or decode)"
+    );
+    metrics::describe_histogram!(
+        "tgi_request_skipped_tokens",
+        metrics::Unit::Count,
+        "Speculated tokens per request"
+    );
+    metrics::describe_histogram!(
+        "tgi_batch_filter_duration",
+        metrics::Unit::Seconds,
+        "Time spent filtering batches and sending generated tokens per method (prefill or decode)"
+    );
+    metrics::describe_histogram!(
+        "tgi_request_queue_duration",
+        metrics::Unit::Seconds,
+        "Time spent in the queue per request"
+    );
+    metrics::describe_histogram!(
+        "tgi_request_validation_duration",
+        metrics::Unit::Seconds,
+        "Time spent validating the request"
+    );
+    metrics::describe_histogram!(
+        "tgi_request_duration",
+        metrics::Unit::Seconds,
+        "Total time spent processing the request"
+    );
+    metrics::describe_histogram!(
+        "tgi_batch_decode_duration",
+        metrics::Unit::Seconds,
+        "Time spent decoding a batch per method (prefill or decode)"
+    );
+    metrics::describe_histogram!(
+        "tgi_request_input_length",
+        metrics::Unit::Count,
+        "Input token length per request"
+    );
+    metrics::describe_histogram!(
+        "tgi_batch_next_size",
+        metrics::Unit::Count,
+        "Batch size of the next batch"
+    );
+
     // CORS layer
     let allow_origin = allow_origin.unwrap_or(AllowOrigin::any());
     let cors_layer = CorsLayer::new()
diff --git a/update_doc.py b/update_doc.py
index e887e1c6..3fb0d314 100644
--- a/update_doc.py
+++ b/update_doc.py
@@ -63,7 +63,7 @@ def check_cli(check: bool):
     final_doc += f"## {header}\n```shell\n{rendered_block}\n```\n"
     block = []
 
-    filename = "docs/source/basic_tutorials/launcher.md"
+    filename = "docs/source/reference/launcher.md"
     if check:
         with open(filename, "r") as f:
             doc = f.read()

From e4201f44cf9f2687027c1830db29607a92e0a9d4 Mon Sep 17 00:00:00 2001
From: Nicolas Patry <patry.nicolas@protonmail.com>
Date: Fri, 16 Aug 2024 21:19:46 +0200
Subject: [PATCH 55/72] All integration tests back everywhere (too many failed
 CI). (#2428)

* All integration tests back everywhere (too many failed CI).

* Upgrade integration tests after 12.4

* Attempt to remove the specifed compute cap.

* Common arch list.

* Punica uses raw ASM which is not valid on 9.0 apparently.
---
 .github/workflows/build.yaml                  |     2 +-
 Dockerfile                                    |     9 +-
 .../test_bloom_560m/test_bloom_560m.json      |    32 +-
 .../test_flash_deepseek_v2_all_params.json    |     8 +-
 .../test_flash_gemma/test_flash_gemma.json    |    32 +-
 .../test_flash_pali_gemma.json                |     4 +-
 .../test_flash_phi_all_params.json            |    10 +-
 .../test_flash_llava_next_load.json           | 65746 ++++++++--------
 .../test_mamba/test_mamba_all_params.json     |    24 +-
 integration-tests/models/test_mamba.py        |     3 +-
 server/custom_kernels/setup.py                |     3 -
 11 files changed, 32936 insertions(+), 32937 deletions(-)

diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
index fd059e70..1f72c46d 100644
--- a/.github/workflows/build.yaml
+++ b/.github/workflows/build.yaml
@@ -172,7 +172,7 @@ jobs:
       group: ${{ needs.build-and-push.outputs.runs_on }}
     if: needs.build-and-push.outputs.runs_on != 'ubuntu-latest'
     env:
-      PYTEST_FLAGS: ${{ (startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/main' || inputs.release-tests == true) && '--release' || '' }}
+      PYTEST_FLAGS: ${{ (startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/main' || inputs.release-tests == true) && '--release' || '--release' }}
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
diff --git a/Dockerfile b/Dockerfile
index ed0c4aa3..4c64a643 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -88,6 +88,7 @@ RUN case ${TARGETPLATFORM} in \
 FROM pytorch-install AS kernel-builder
 
 ARG MAX_JOBS=8
+ENV TORCH_CUDA_ARCH_LIST="8.0;8.6;9.0+PTX"
 
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
         ninja-build cmake \
@@ -118,7 +119,7 @@ FROM kernel-builder AS exllama-kernels-builder
 WORKDIR /usr/src
 COPY server/exllama_kernels/ .
 
-RUN TORCH_CUDA_ARCH_LIST="8.0;8.6+PTX" python setup.py build
+RUN python setup.py build
 
 # Build Transformers exllama kernels
 FROM kernel-builder AS exllamav2-kernels-builder
@@ -126,21 +127,21 @@ WORKDIR /usr/src
 COPY server/Makefile-exllamav2/ Makefile
 
 # Build specific version of transformers
-RUN TORCH_CUDA_ARCH_LIST="8.0;8.6+PTX" make build-exllamav2
+RUN make build-exllamav2
 
 # Build Transformers awq kernels
 FROM kernel-builder AS awq-kernels-builder
 WORKDIR /usr/src
 COPY server/Makefile-awq Makefile
 # Build specific version of transformers
-RUN TORCH_CUDA_ARCH_LIST="8.0;8.6+PTX" make build-awq
+RUN make build-awq
 
 # Build eetq kernels
 FROM kernel-builder AS eetq-kernels-builder
 WORKDIR /usr/src
 COPY server/Makefile-eetq Makefile
 # Build specific version of transformers
-RUN TORCH_CUDA_ARCH_LIST="8.0;8.6+PTX" make build-eetq
+RUN make build-eetq
 
 # Build Lorax Punica kernels
 FROM kernel-builder AS lorax-punica-builder
diff --git a/integration-tests/models/__snapshots__/test_bloom_560m/test_bloom_560m.json b/integration-tests/models/__snapshots__/test_bloom_560m/test_bloom_560m.json
index 5d0eeef6..54c66408 100644
--- a/integration-tests/models/__snapshots__/test_bloom_560m/test_bloom_560m.json
+++ b/integration-tests/models/__snapshots__/test_bloom_560m/test_bloom_560m.json
@@ -11,52 +11,52 @@
       },
       {
         "id": 49833,
-        "logprob": -10.546875,
+        "logprob": -10.5703125,
         "text": " dég"
       },
       {
         "id": 21543,
-        "logprob": -0.18457031,
+        "logprob": -0.14746094,
         "text": "uster"
       },
       {
         "id": 447,
-        "logprob": -1.9287109,
+        "logprob": -1.9277344,
         "text": " un"
       },
       {
         "id": 46341,
-        "logprob": -15.4296875,
+        "logprob": -15.421875,
         "text": " ort"
       },
       {
         "id": 35567,
-        "logprob": -7.578125,
+        "logprob": -7.5820312,
         "text": "olan"
       },
       {
         "id": 15,
-        "logprob": -1.4003906,
+        "logprob": -1.4013672,
         "text": ","
       },
       {
         "id": 1669,
-        "logprob": -1.5439453,
+        "logprob": -1.5595703,
         "text": " il"
       },
       {
         "id": 11580,
-        "logprob": -0.93896484,
+        "logprob": -0.9428711,
         "text": " faut"
       },
       {
         "id": 3913,
-        "logprob": -3.7207031,
+        "logprob": -3.703125,
         "text": " tout"
       },
       {
         "id": 39261,
-        "logprob": -1.5742188,
+        "logprob": -1.7763672,
         "text": " d'abord"
       }
     ],
@@ -64,19 +64,19 @@
     "tokens": [
       {
         "id": 578,
-        "logprob": -1.6474609,
+        "logprob": -1.7822266,
         "special": false,
         "text": " le"
       },
       {
         "id": 5608,
-        "logprob": -2.4707031,
+        "logprob": -2.4882812,
         "special": false,
         "text": " faire"
       },
       {
         "id": 7735,
-        "logprob": -2.4355469,
+        "logprob": -2.4199219,
         "special": false,
         "text": " fond"
       },
@@ -94,13 +94,13 @@
       },
       {
         "id": 366,
-        "logprob": -1.1953125,
+        "logprob": -1.1308594,
         "special": false,
         "text": " la"
       },
       {
         "id": 48844,
-        "logprob": -1.7978516,
+        "logprob": -1.7900391,
         "special": false,
         "text": " cass"
       },
@@ -118,7 +118,7 @@
       },
       {
         "id": 2940,
-        "logprob": -1.9335938,
+        "logprob": -1.9306641,
         "special": false,
         "text": " avec"
       }
diff --git a/integration-tests/models/__snapshots__/test_flash_deepseek_v2/test_flash_deepseek_v2_all_params.json b/integration-tests/models/__snapshots__/test_flash_deepseek_v2/test_flash_deepseek_v2_all_params.json
index 3ac8d050..6b45cf6b 100644
--- a/integration-tests/models/__snapshots__/test_flash_deepseek_v2/test_flash_deepseek_v2_all_params.json
+++ b/integration-tests/models/__snapshots__/test_flash_deepseek_v2/test_flash_deepseek_v2_all_params.json
@@ -11,7 +11,7 @@
       },
       {
         "id": 3533,
-        "logprob": -9.5625,
+        "logprob": -9.625,
         "text": "Test"
       },
       {
@@ -24,13 +24,13 @@
     "tokens": [
       {
         "id": 2143,
-        "logprob": -1.8203125,
+        "logprob": -1.828125,
         "special": false,
         "text": " sent"
       },
       {
         "id": 10081,
-        "logprob": -0.55078125,
+        "logprob": -0.41210938,
         "special": false,
         "text": " successfully"
       },
@@ -42,7 +42,7 @@
       },
       {
         "id": 100001,
-        "logprob": -0.12695312,
+        "logprob": -0.16015625,
         "special": true,
         "text": "<｜end▁of▁sentence｜>"
       }
diff --git a/integration-tests/models/__snapshots__/test_flash_gemma/test_flash_gemma.json b/integration-tests/models/__snapshots__/test_flash_gemma/test_flash_gemma.json
index 96f2ce17..8829f9fe 100644
--- a/integration-tests/models/__snapshots__/test_flash_gemma/test_flash_gemma.json
+++ b/integration-tests/models/__snapshots__/test_flash_gemma/test_flash_gemma.json
@@ -24,13 +24,13 @@
     "tokens": [
       {
         "id": 1736,
-        "logprob": -2.046875,
+        "logprob": -2.03125,
         "special": false,
         "text": " form"
       },
       {
         "id": 109,
-        "logprob": -1.8828125,
+        "logprob": -1.8671875,
         "special": false,
         "text": "\n\n"
       },
@@ -42,48 +42,48 @@
       },
       {
         "id": 2121,
-        "logprob": -1.78125,
+        "logprob": -1.8125,
         "special": false,
         "text": " test"
       },
       {
         "id": 3853,
-        "logprob": -0.23632812,
+        "logprob": -0.24121094,
         "special": false,
         "text": " request"
       },
       {
         "id": 1736,
-        "logprob": -0.09326172,
+        "logprob": -0.100097656,
         "special": false,
         "text": " form"
       },
       {
         "id": 603,
-        "logprob": -0.8828125,
+        "logprob": -0.9453125,
         "special": false,
         "text": " is"
       },
       {
-        "id": 1671,
-        "logprob": -1.6171875,
+        "id": 476,
+        "logprob": -1.703125,
         "special": false,
-        "text": " used"
+        "text": " a"
       },
       {
-        "id": 577,
-        "logprob": -0.390625,
+        "id": 4551,
+        "logprob": -2.453125,
         "special": false,
-        "text": " to"
+        "text": " document"
       },
       {
-        "id": 3853,
-        "logprob": -1.2265625,
+        "id": 674,
+        "logprob": -0.796875,
         "special": false,
-        "text": " request"
+        "text": " that"
       }
     ],
     "top_tokens": null
   },
-  "generated_text": " form\n\nThe test request form is used to request"
+  "generated_text": " form\n\nThe test request form is a document that"
 }
diff --git a/integration-tests/models/__snapshots__/test_flash_pali_gemma/test_flash_pali_gemma.json b/integration-tests/models/__snapshots__/test_flash_pali_gemma/test_flash_pali_gemma.json
index 037e0b16..0096f807 100644
--- a/integration-tests/models/__snapshots__/test_flash_pali_gemma/test_flash_pali_gemma.json
+++ b/integration-tests/models/__snapshots__/test_flash_pali_gemma/test_flash_pali_gemma.json
@@ -8,13 +8,13 @@
     "tokens": [
       {
         "id": 54901,
-        "logprob": -0.72753906,
+        "logprob": -0.84765625,
         "special": false,
         "text": "beach"
       },
       {
         "id": 1,
-        "logprob": -0.011009216,
+        "logprob": -0.008666992,
         "special": true,
         "text": "<eos>"
       }
diff --git a/integration-tests/models/__snapshots__/test_flash_phi/test_flash_phi_all_params.json b/integration-tests/models/__snapshots__/test_flash_phi/test_flash_phi_all_params.json
index 221ff13d..b407ef71 100644
--- a/integration-tests/models/__snapshots__/test_flash_phi/test_flash_phi_all_params.json
+++ b/integration-tests/models/__snapshots__/test_flash_phi/test_flash_phi_all_params.json
@@ -19,25 +19,25 @@
     "tokens": [
       {
         "id": 284,
-        "logprob": -0.19421387,
+        "logprob": -0.28955078,
         "special": false,
         "text": " to"
       },
       {
         "id": 3758,
-        "logprob": -0.62597656,
+        "logprob": -0.7739258,
         "special": false,
         "text": " send"
       },
       {
         "id": 1366,
-        "logprob": -0.87060547,
+        "logprob": -0.85253906,
         "special": false,
         "text": " data"
       },
       {
         "id": 625,
-        "logprob": -0.88427734,
+        "logprob": -0.8984375,
         "special": false,
         "text": " over"
       },
@@ -49,7 +49,7 @@
       },
       {
         "id": 3127,
-        "logprob": -1.9462891,
+        "logprob": -1.9404297,
         "special": false,
         "text": " network"
       }
diff --git a/integration-tests/models/__snapshots__/test_llava_next/test_flash_llava_next_load.json b/integration-tests/models/__snapshots__/test_llava_next/test_flash_llava_next_load.json
index 2007c0f2..50ce2e31 100644
--- a/integration-tests/models/__snapshots__/test_llava_next/test_flash_llava_next_load.json
+++ b/integration-tests/models/__snapshots__/test_llava_next/test_flash_llava_next_load.json
@@ -12,7 +12,7 @@
         },
         {
           "id": 1247,
-          "logprob": -2.3886719,
+          "logprob": -2.390625,
           "text": "User"
         },
         {
@@ -27,12 +27,12 @@
         },
         {
           "id": 32000,
-          "logprob": -10.671875,
+          "logprob": -10.6953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.7109375,
+          "logprob": -15.828125,
           "text": "<image>"
         },
         {
@@ -42,82 +42,82 @@
         },
         {
           "id": 32000,
-          "logprob": -10.0234375,
+          "logprob": -10.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.1328125,
+          "logprob": -10.140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.421875,
+          "logprob": -10.3125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.90625,
+          "logprob": -12.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.59375,
+          "logprob": -15.4921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.828125,
+          "logprob": -13.8671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.390625,
+          "logprob": -11.3671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.1171875,
+          "logprob": -10.1015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.1640625,
+          "logprob": -10.1796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.234375,
+          "logprob": -10.2421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.3984375,
+          "logprob": -10.4609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.015625,
+          "logprob": -14.34375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0859375,
+          "logprob": -13.1796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2734375,
+          "logprob": -13.328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.359375,
+          "logprob": -14.3515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0390625,
+          "logprob": -11.0546875,
           "text": "<image>"
         },
         {
@@ -127,12 +127,12 @@
         },
         {
           "id": 32000,
-          "logprob": -10.5234375,
+          "logprob": -10.515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.4765625,
+          "logprob": -10.4453125,
           "text": "<image>"
         },
         {
@@ -142,22 +142,22 @@
         },
         {
           "id": 32000,
-          "logprob": -13.6171875,
+          "logprob": -13.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.359375,
+          "logprob": -11.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.8359375,
+          "logprob": -10.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -17.34375,
+          "logprob": -17.234375,
           "text": "<image>"
         },
         {
@@ -172,12 +172,12 @@
         },
         {
           "id": 32000,
-          "logprob": -10.640625,
+          "logprob": -10.6484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -18.390625,
+          "logprob": -17.984375,
           "text": "<image>"
         },
         {
@@ -187,17 +187,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.5625,
+          "logprob": -11.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -9.875,
+          "logprob": -9.8671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.7734375,
+          "logprob": -10.7578125,
           "text": "<image>"
         },
         {
@@ -207,32 +207,32 @@
         },
         {
           "id": 32000,
-          "logprob": -10.96875,
+          "logprob": -10.984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.609375,
+          "logprob": -10.625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.09375,
+          "logprob": -11.109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.5078125,
+          "logprob": -10.5,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4453125,
+          "logprob": -11.4609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.59375,
+          "logprob": -13.09375,
           "text": "<image>"
         },
         {
@@ -242,132 +242,87 @@
         },
         {
           "id": 32000,
-          "logprob": -10.5625,
+          "logprob": -10.5390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.640625,
+          "logprob": -10.625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9765625,
+          "logprob": -10.9296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.765625,
+          "logprob": -11.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3671875,
+          "logprob": -12.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0234375,
+          "logprob": -12.125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.59375,
+          "logprob": -10.578125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.7421875,
+          "logprob": -10.7265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0625,
+          "logprob": -11.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.3828125,
+          "logprob": -10.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.171875,
+          "logprob": -11.1796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0234375,
+          "logprob": -11.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -18.40625,
+          "logprob": -18.703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9921875,
+          "logprob": -11.015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.7109375,
+          "logprob": -15.6796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.15625,
+          "logprob": -12.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.140625,
+          "logprob": -10.4140625,
           "text": "<image>"
         },
         {
@@ -377,12 +332,42 @@
         },
         {
           "id": 32000,
-          "logprob": -10.7890625,
+          "logprob": -11.8046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4296875,
+          "logprob": -10.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.140625,
           "text": "<image>"
         },
         {
@@ -392,12 +377,27 @@
         },
         {
           "id": 32000,
-          "logprob": -10.796875,
+          "logprob": -10.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1640625,
+          "logprob": -11.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.15625,
           "text": "<image>"
         },
         {
@@ -407,27 +407,27 @@
         },
         {
           "id": 32000,
-          "logprob": -15.4453125,
+          "logprob": -15.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.2109375,
+          "logprob": -10.2265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.09375,
+          "logprob": -11.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.6796875,
+          "logprob": -14.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.3671875,
+          "logprob": -10.375,
           "text": "<image>"
         },
         {
@@ -442,22 +442,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.484375,
+          "logprob": -12.4921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.09375,
+          "logprob": -12.1015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.96875,
+          "logprob": -11.0859375,
           "text": "<image>"
         },
         {
@@ -465,6 +460,11 @@
           "logprob": -10.9765625,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -10.921875,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -11.0234375,
@@ -480,11 +480,6 @@
           "logprob": -11.1484375,
           "text": "<image>"
         },
-        {
-          "id": 32000,
-          "logprob": -10.703125,
-          "text": "<image>"
-        },
         {
           "id": 32000,
           "logprob": -10.71875,
@@ -492,17 +487,22 @@
         },
         {
           "id": 32000,
-          "logprob": -11.6875,
+          "logprob": -10.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.8984375,
+          "logprob": -11.6796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2890625,
+          "logprob": -10.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.375,
           "text": "<image>"
         },
         {
@@ -512,12 +512,12 @@
         },
         {
           "id": 32000,
-          "logprob": -10.640625,
+          "logprob": -10.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.7109375,
+          "logprob": -10.7265625,
           "text": "<image>"
         },
         {
@@ -532,17 +532,17 @@
         },
         {
           "id": 32000,
-          "logprob": -10.6875,
+          "logprob": -10.6796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.84375,
+          "logprob": -11.828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5078125,
+          "logprob": -11.5390625,
           "text": "<image>"
         },
         {
@@ -552,22 +552,22 @@
         },
         {
           "id": 32000,
-          "logprob": -10.9609375,
+          "logprob": -10.9765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4921875,
+          "logprob": -12.5,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5546875,
+          "logprob": -11.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.2265625,
+          "logprob": -11.2421875,
           "text": "<image>"
         },
         {
@@ -577,7 +577,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.21875,
+          "logprob": -11.2265625,
           "text": "<image>"
         },
         {
@@ -587,52 +587,52 @@
         },
         {
           "id": 32000,
-          "logprob": -11.0,
+          "logprob": -10.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0234375,
+          "logprob": -12.046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.09375,
+          "logprob": -11.109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.046875,
+          "logprob": -14.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.921875,
+          "logprob": -10.8984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9609375,
+          "logprob": -10.953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9140625,
+          "logprob": -10.9296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8125,
+          "logprob": -11.78125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.140625,
+          "logprob": -15.1484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9609375,
+          "logprob": -10.96875,
           "text": "<image>"
         },
         {
@@ -642,72 +642,72 @@
         },
         {
           "id": 32000,
-          "logprob": -10.8828125,
+          "logprob": -10.890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9609375,
+          "logprob": -10.9453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.46875,
+          "logprob": -11.453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5234375,
+          "logprob": -13.09375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.328125,
+          "logprob": -12.484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.375,
+          "logprob": -11.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3515625,
+          "logprob": -12.3359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.578125,
+          "logprob": -11.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5078125,
+          "logprob": -12.5,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.640625,
+          "logprob": -12.6328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1953125,
+          "logprob": -11.203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9921875,
+          "logprob": -11.9765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.921875,
+          "logprob": -10.9296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.7578125,
+          "logprob": -10.7421875,
           "text": "<image>"
         },
         {
@@ -717,12 +717,12 @@
         },
         {
           "id": 32000,
-          "logprob": -15.015625,
+          "logprob": -15.0234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7734375,
+          "logprob": -12.75,
           "text": "<image>"
         },
         {
@@ -737,27 +737,27 @@
         },
         {
           "id": 32000,
-          "logprob": -11.2890625,
+          "logprob": -11.265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.34375,
+          "logprob": -13.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.6953125,
+          "logprob": -10.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.59375,
+          "logprob": -10.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6015625,
+          "logprob": -11.7578125,
           "text": "<image>"
         },
         {
@@ -772,7 +772,7 @@
         },
         {
           "id": 32000,
-          "logprob": -10.5390625,
+          "logprob": -10.5859375,
           "text": "<image>"
         },
         {
@@ -782,12 +782,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.9765625,
+          "logprob": -17.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4609375,
+          "logprob": -11.4765625,
           "text": "<image>"
         },
         {
@@ -797,12 +797,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.015625,
+          "logprob": -12.0078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.84375,
+          "logprob": -12.8359375,
           "text": "<image>"
         },
         {
@@ -812,12 +812,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.4375,
+          "logprob": -12.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.671875,
+          "logprob": -12.6875,
           "text": "<image>"
         },
         {
@@ -827,57 +827,27 @@
         },
         {
           "id": 32000,
-          "logprob": -12.875,
+          "logprob": -12.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2578125,
+          "logprob": -12.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.359375,
+          "logprob": -10.3359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.375,
+          "logprob": -13.46875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.7421875,
+          "logprob": -11.8671875,
           "text": "<image>"
         },
         {
@@ -887,7 +857,32 @@
         },
         {
           "id": 32000,
-          "logprob": -11.0390625,
+          "logprob": -11.0078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8203125,
           "text": "<image>"
         },
         {
@@ -897,17 +892,22 @@
         },
         {
           "id": 32000,
-          "logprob": -10.6171875,
+          "logprob": -10.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2421875,
+          "logprob": -10.6328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.8359375,
+          "logprob": -12.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8203125,
           "text": "<image>"
         },
         {
@@ -927,127 +927,62 @@
         },
         {
           "id": 32000,
-          "logprob": -11.0,
+          "logprob": -10.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7734375,
+          "logprob": -12.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2109375,
+          "logprob": -12.21875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4296875,
+          "logprob": -11.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2421875,
+          "logprob": -12.828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3046875,
+          "logprob": -11.328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.2890625,
+          "logprob": -10.296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.8203125,
+          "logprob": -10.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9140625,
+          "logprob": -11.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.2421875,
+          "logprob": -11.21875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.234375,
+          "logprob": -11.2109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4375,
+          "logprob": -11.5078125,
           "text": "<image>"
         },
         {
@@ -1057,27 +992,12 @@
         },
         {
           "id": 32000,
-          "logprob": -14.203125,
+          "logprob": -10.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.59375,
+          "logprob": -10.3515625,
           "text": "<image>"
         },
         {
@@ -1087,102 +1007,22 @@
         },
         {
           "id": 32000,
-          "logprob": -10.90625,
+          "logprob": -10.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.2578125,
+          "logprob": -10.828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.921875,
+          "logprob": -11.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.890625,
+          "logprob": -11.9375,
           "text": "<image>"
         },
         {
@@ -1192,112 +1032,32 @@
         },
         {
           "id": 32000,
-          "logprob": -10.9609375,
+          "logprob": -12.3046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.7890625,
+          "logprob": -11.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4453125,
+          "logprob": -12.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.390625,
+          "logprob": -13.3984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1640625,
+          "logprob": -11.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0703125,
+          "logprob": -14.1875,
           "text": "<image>"
         },
         {
@@ -1307,72 +1067,162 @@
         },
         {
           "id": 32000,
-          "logprob": -12.859375,
+          "logprob": -12.421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1875,
+          "logprob": -10.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.171875,
+          "logprob": -10.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6640625,
+          "logprob": -10.6015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1953125,
+          "logprob": -10.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1328125,
+          "logprob": -11.2265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1875,
+          "logprob": -10.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9453125,
+          "logprob": -10.9765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8515625,
+          "logprob": -12.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.203125,
+          "logprob": -10.8671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1796875,
+          "logprob": -10.8515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9609375,
+          "logprob": -11.8984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.703125,
+          "logprob": -10.7421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8515625,
+          "logprob": -13.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1796875,
           "text": "<image>"
         },
         {
@@ -1380,11 +1230,161 @@
           "logprob": -11.75,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -10.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.640625,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -11.8359375,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -11.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.828125,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -12.1953125,
@@ -1392,32 +1392,32 @@
         },
         {
           "id": 32000,
-          "logprob": -12.5078125,
+          "logprob": -12.4921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.546875,
+          "logprob": -11.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.078125,
+          "logprob": -12.0859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.2421875,
+          "logprob": -11.265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6640625,
+          "logprob": -11.6953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2109375,
+          "logprob": -12.21875,
           "text": "<image>"
         },
         {
@@ -1437,27 +1437,7 @@
         },
         {
           "id": 32000,
-          "logprob": -15.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3046875,
+          "logprob": -15.03125,
           "text": "<image>"
         },
         {
@@ -1467,32 +1447,52 @@
         },
         {
           "id": 32000,
-          "logprob": -12.2265625,
+          "logprob": -11.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9453125,
+          "logprob": -13.1171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4921875,
+          "logprob": -12.296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.8828125,
+          "logprob": -12.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3125,
+          "logprob": -12.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8984375,
+          "logprob": -12.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9140625,
           "text": "<image>"
         },
         {
@@ -1502,12 +1502,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8125,
+          "logprob": -11.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.90625,
+          "logprob": -11.9140625,
           "text": "<image>"
         },
         {
@@ -1527,12 +1527,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.140625,
+          "logprob": -13.1484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9765625,
+          "logprob": -11.9921875,
           "text": "<image>"
         },
         {
@@ -1547,7 +1547,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.84375,
+          "logprob": -11.8359375,
           "text": "<image>"
         },
         {
@@ -1562,17 +1562,22 @@
         },
         {
           "id": 32000,
-          "logprob": -13.1953125,
+          "logprob": -13.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.875,
+          "logprob": -11.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.6015625,
+          "logprob": -13.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6484375,
           "text": "<image>"
         },
         {
@@ -1582,27 +1587,22 @@
         },
         {
           "id": 32000,
-          "logprob": -13.671875,
+          "logprob": -11.4609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5390625,
+          "logprob": -13.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.59375,
+          "logprob": -12.4375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5703125,
+          "logprob": -14.6328125,
           "text": "<image>"
         },
         {
@@ -1612,57 +1612,57 @@
         },
         {
           "id": 32000,
-          "logprob": -13.4140625,
+          "logprob": -13.3984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7890625,
+          "logprob": -11.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6328125,
+          "logprob": -11.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.4296875,
+          "logprob": -14.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.53125,
+          "logprob": -13.5390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.515625,
+          "logprob": -14.5078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7265625,
+          "logprob": -12.7109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.609375,
+          "logprob": -11.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.171875,
+          "logprob": -13.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.109375,
+          "logprob": -13.09375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8828125,
+          "logprob": -12.875,
           "text": "<image>"
         },
         {
@@ -1672,37 +1672,37 @@
         },
         {
           "id": 32000,
-          "logprob": -12.671875,
+          "logprob": -12.6484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.7109375,
+          "logprob": -13.6953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4296875,
+          "logprob": -12.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.296875,
+          "logprob": -12.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1796875,
+          "logprob": -13.109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2421875,
+          "logprob": -12.1796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.828125,
+          "logprob": -14.875,
           "text": "<image>"
         },
         {
@@ -1712,7 +1712,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.3359375,
+          "logprob": -11.34375,
           "text": "<image>"
         },
         {
@@ -1732,67 +1732,7 @@
         },
         {
           "id": 32000,
-          "logprob": -15.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0,
+          "logprob": -15.0859375,
           "text": "<image>"
         },
         {
@@ -1802,57 +1742,117 @@
         },
         {
           "id": 32000,
-          "logprob": -16.21875,
+          "logprob": -11.875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6015625,
+          "logprob": -13.25,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.796875,
+          "logprob": -13.5078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7265625,
+          "logprob": -13.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9453125,
+          "logprob": -14.265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9765625,
+          "logprob": -14.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.71875,
+          "logprob": -13.2421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.6953125,
+          "logprob": -12.28125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3359375,
+          "logprob": -13.8046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3203125,
+          "logprob": -12.703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6328125,
+          "logprob": -12.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.609375,
           "text": "<image>"
         },
         {
@@ -1862,17 +1862,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1875,
+          "logprob": -14.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.046875,
+          "logprob": -12.0234375,
           "text": "<image>"
         },
         {
@@ -1887,17 +1887,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.8828125,
+          "logprob": -12.890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.296875,
+          "logprob": -12.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2109375,
+          "logprob": -12.21875,
           "text": "<image>"
         },
         {
@@ -1907,27 +1907,27 @@
         },
         {
           "id": 32000,
-          "logprob": -15.1875,
+          "logprob": -15.1953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5390625,
+          "logprob": -11.546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.421875,
+          "logprob": -14.4375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2890625,
+          "logprob": -12.265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2265625,
+          "logprob": -12.21875,
           "text": "<image>"
         },
         {
@@ -1942,12 +1942,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.5859375,
+          "logprob": -13.5390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.859375,
+          "logprob": -11.8515625,
           "text": "<image>"
         },
         {
@@ -1957,12 +1957,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.1015625,
+          "logprob": -13.046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.9453125,
+          "logprob": -13.953125,
           "text": "<image>"
         },
         {
@@ -1977,47 +1977,47 @@
         },
         {
           "id": 32000,
-          "logprob": -12.734375,
+          "logprob": -12.7109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.203125,
+          "logprob": -12.1796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.59375,
+          "logprob": -12.578125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3984375,
+          "logprob": -11.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5,
+          "logprob": -13.4609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.765625,
+          "logprob": -11.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2265625,
+          "logprob": -13.21875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7578125,
+          "logprob": -11.75,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.3515625,
+          "logprob": -15.34375,
           "text": "<image>"
         },
         {
@@ -2032,22 +2032,42 @@
         },
         {
           "id": 32000,
-          "logprob": -13.3671875,
+          "logprob": -13.3515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.90625,
+          "logprob": -11.8828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5625,
+          "logprob": -12.578125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3203125,
+          "logprob": -13.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.125,
           "text": "<image>"
         },
         {
@@ -2057,27 +2077,7 @@
         },
         {
           "id": 32000,
-          "logprob": -10.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9921875,
+          "logprob": -11.984375,
           "text": "<image>"
         },
         {
@@ -2092,27 +2092,27 @@
         },
         {
           "id": 32000,
-          "logprob": -12.9765625,
+          "logprob": -12.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4921875,
+          "logprob": -12.484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0625,
+          "logprob": -12.046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1796875,
+          "logprob": -12.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3359375,
+          "logprob": -13.296875,
           "text": "<image>"
         },
         {
@@ -2122,32 +2122,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.375,
+          "logprob": -12.3671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.984375,
+          "logprob": -13.1328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9453125,
+          "logprob": -13.625,
           "text": "<image>"
         },
         {
@@ -2157,87 +2142,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.3203125,
+          "logprob": -11.75,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3125,
+          "logprob": -11.9140625,
           "text": "<image>"
         },
         {
@@ -2247,12 +2157,102 @@
         },
         {
           "id": 32000,
-          "logprob": -11.3984375,
+          "logprob": -12.3359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.84375,
+          "logprob": -12.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8359375,
           "text": "<image>"
         },
         {
@@ -2262,52 +2262,52 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6328125,
+          "logprob": -12.6171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.875,
+          "logprob": -13.84375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.109375,
+          "logprob": -12.09375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1171875,
+          "logprob": -12.140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4921875,
+          "logprob": -11.5078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.2890625,
+          "logprob": -11.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.15625,
+          "logprob": -12.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.59375,
+          "logprob": -13.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8046875,
+          "logprob": -11.7890625,
           "text": "<image>"
         },
         {
@@ -2317,22 +2317,22 @@
         },
         {
           "id": 32000,
-          "logprob": -13.1015625,
+          "logprob": -13.0625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2265625,
+          "logprob": -13.1875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2109375,
+          "logprob": -13.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4609375,
+          "logprob": -12.46875,
           "text": "<image>"
         },
         {
@@ -2342,12 +2342,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
+          "logprob": -12.6953125,
           "text": "<image>"
         },
         {
@@ -2355,6 +2350,11 @@
           "logprob": -11.6875,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -11.6953125,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -11.7109375,
@@ -2362,27 +2362,27 @@
         },
         {
           "id": 32000,
-          "logprob": -15.390625,
+          "logprob": -15.3359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.953125,
+          "logprob": -11.9453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.140625,
+          "logprob": -16.125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.4921875,
+          "logprob": -15.4609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.9296875,
+          "logprob": -13.921875,
           "text": "<image>"
         },
         {
@@ -2397,7 +2397,7 @@
         },
         {
           "id": 32000,
-          "logprob": -15.984375,
+          "logprob": -16.0,
           "text": "<image>"
         },
         {
@@ -2407,37 +2407,37 @@
         },
         {
           "id": 32000,
-          "logprob": -12.8671875,
+          "logprob": -12.828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4921875,
+          "logprob": -12.5078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7421875,
+          "logprob": -11.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1875,
+          "logprob": -14.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3515625,
+          "logprob": -11.34375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.71875,
+          "logprob": -11.7265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0625,
+          "logprob": -13.046875,
           "text": "<image>"
         },
         {
@@ -2447,12 +2447,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.421875,
+          "logprob": -12.4375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.2734375,
+          "logprob": -14.265625,
           "text": "<image>"
         },
         {
@@ -2462,77 +2462,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.390625,
+          "logprob": -12.6953125,
           "text": "<image>"
         },
         {
@@ -2542,27 +2472,97 @@
         },
         {
           "id": 32000,
-          "logprob": -13.078125,
+          "logprob": -12.984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4296875,
+          "logprob": -13.2890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.828125,
+          "logprob": -13.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8046875,
+          "logprob": -11.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6484375,
+          "logprob": -10.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.625,
           "text": "<image>"
         },
         {
@@ -2577,27 +2577,27 @@
         },
         {
           "id": 32000,
-          "logprob": -14.921875,
+          "logprob": -14.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.578125,
+          "logprob": -12.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.78125,
+          "logprob": -12.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3984375,
+          "logprob": -11.375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.0546875,
+          "logprob": -14.03125,
           "text": "<image>"
         },
         {
@@ -2617,12 +2617,12 @@
         },
         {
           "id": 32000,
-          "logprob": -14.5234375,
+          "logprob": -14.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.625,
+          "logprob": -12.609375,
           "text": "<image>"
         },
         {
@@ -2632,12 +2632,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.6015625,
+          "logprob": -13.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.28125,
+          "logprob": -13.2109375,
           "text": "<image>"
         },
         {
@@ -2657,17 +2657,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.0,
+          "logprob": -12.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6640625,
+          "logprob": -12.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.46875,
+          "logprob": -11.484375,
           "text": "<image>"
         },
         {
@@ -2677,12 +2677,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8828125,
+          "logprob": -11.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1015625,
+          "logprob": -13.078125,
           "text": "<image>"
         },
         {
@@ -2697,7 +2697,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.5234375,
+          "logprob": -11.546875,
           "text": "<image>"
         },
         {
@@ -2712,7 +2712,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.4921875,
+          "logprob": -12.5,
           "text": "<image>"
         },
         {
@@ -2727,122 +2727,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.7890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1484375,
+          "logprob": -12.078125,
           "text": "<image>"
         },
         {
@@ -2852,132 +2737,102 @@
         },
         {
           "id": 32000,
-          "logprob": -13.3828125,
+          "logprob": -13.2265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3203125,
+          "logprob": -14.8046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.34375,
+          "logprob": -12.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -11.8671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.140625,
+          "logprob": -13.1484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9921875,
+          "logprob": -12.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.15625,
+          "logprob": -10.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.84375,
+          "logprob": -14.6875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.21875,
+          "logprob": -15.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.46875,
+          "logprob": -15.453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1484375,
+          "logprob": -11.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.515625,
+          "logprob": -14.484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.84375,
+          "logprob": -12.4375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.015625,
+          "logprob": -16.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.28125,
+          "logprob": -14.875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6015625,
+          "logprob": -16.109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3984375,
+          "logprob": -13.2890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.375,
+          "logprob": -12.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5625,
+          "logprob": -11.8359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -9.9375,
+          "logprob": -12.9921875,
           "text": "<image>"
         },
         {
@@ -2987,17 +2842,162 @@
         },
         {
           "id": 32000,
-          "logprob": -12.203125,
+          "logprob": -12.1484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.2890625,
+          "logprob": -13.84375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1953125,
+          "logprob": -13.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -9.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.15625,
           "text": "<image>"
         },
         {
@@ -3010,21 +3010,6 @@
           "logprob": -12.28125,
           "text": "<image>"
         },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
         {
           "id": 32000,
           "logprob": -11.8984375,
@@ -3032,52 +3017,67 @@
         },
         {
           "id": 32000,
-          "logprob": -10.5859375,
+          "logprob": -14.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9921875,
+          "logprob": -14.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9921875,
+          "logprob": -11.8671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1015625,
+          "logprob": -10.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.5390625,
+          "logprob": -13.0,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.2578125,
+          "logprob": -12.8984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1171875,
+          "logprob": -12.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2421875,
+          "logprob": -15.765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5,
+          "logprob": -15.203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7265625,
+          "logprob": -14.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8359375,
           "text": "<image>"
         },
         {
@@ -3087,7 +3087,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.0,
+          "logprob": -11.0078125,
           "text": "<image>"
         },
         {
@@ -3097,22 +3097,22 @@
         },
         {
           "id": 32000,
-          "logprob": -13.28125,
+          "logprob": -13.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7734375,
+          "logprob": -11.7578125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0625,
+          "logprob": -13.09375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.671875,
+          "logprob": -13.6796875,
           "text": "<image>"
         },
         {
@@ -3122,7 +3122,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.3828125,
+          "logprob": -12.421875,
           "text": "<image>"
         },
         {
@@ -3132,22 +3132,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.25,
+          "logprob": -12.2890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.9453125,
+          "logprob": -11.921875,
           "text": "<image>"
         },
         {
@@ -3157,7 +3147,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.9375,
+          "logprob": -15.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
           "text": "<image>"
         },
         {
@@ -3167,37 +3167,37 @@
         },
         {
           "id": 32000,
-          "logprob": -11.3203125,
+          "logprob": -11.328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4921875,
+          "logprob": -11.5,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.359375,
+          "logprob": -11.3671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3359375,
+          "logprob": -11.3515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0546875,
+          "logprob": -12.0625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.359375,
+          "logprob": -12.3359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -9.6953125,
+          "logprob": -9.7109375,
           "text": "<image>"
         },
         {
@@ -3207,27 +3207,27 @@
         },
         {
           "id": 32000,
-          "logprob": -14.3203125,
+          "logprob": -14.2421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9609375,
+          "logprob": -11.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0859375,
+          "logprob": -12.09375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2109375,
+          "logprob": -12.1796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4921875,
+          "logprob": -11.5234375,
           "text": "<image>"
         },
         {
@@ -3242,7 +3242,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.0546875,
           "text": "<image>"
         },
         {
@@ -3257,22 +3257,22 @@
         },
         {
           "id": 32000,
-          "logprob": -11.9921875,
+          "logprob": -12.0,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2265625,
+          "logprob": -12.1953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9921875,
+          "logprob": -12.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6796875,
+          "logprob": -12.7109375,
           "text": "<image>"
         },
         {
@@ -3282,7 +3282,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.5703125,
+          "logprob": -13.5546875,
           "text": "<image>"
         },
         {
@@ -3297,17 +3297,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.9375,
+          "logprob": -11.953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.9453125,
+          "logprob": -13.8046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.984375,
+          "logprob": -11.9921875,
           "text": "<image>"
         },
         {
@@ -3317,17 +3317,17 @@
         },
         {
           "id": 32000,
-          "logprob": -10.03125,
+          "logprob": -10.015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7265625,
+          "logprob": -11.71875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.7890625,
+          "logprob": -13.8125,
           "text": "<image>"
         },
         {
@@ -3337,42 +3337,42 @@
         },
         {
           "id": 32000,
-          "logprob": -12.671875,
+          "logprob": -12.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.59375,
+          "logprob": -12.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1171875,
+          "logprob": -12.1015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4609375,
+          "logprob": -12.453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3046875,
+          "logprob": -12.2890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4765625,
+          "logprob": -12.46875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6328125,
+          "logprob": -12.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.03125,
+          "logprob": -12.015625,
           "text": "<image>"
         },
         {
@@ -3382,132 +3382,32 @@
         },
         {
           "id": 32000,
-          "logprob": -14.5078125,
+          "logprob": -14.421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2265625,
+          "logprob": -13.25,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.546875,
+          "logprob": -16.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.4765625,
+          "logprob": -14.484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.71875,
+          "logprob": -13.7421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.7890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.671875,
+          "logprob": -13.46875,
           "text": "<image>"
         },
         {
@@ -3517,57 +3417,97 @@
         },
         {
           "id": 32000,
-          "logprob": -13.40625,
+          "logprob": -12.8671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.265625,
+          "logprob": -12.4921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.015625,
+          "logprob": -13.2265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5390625,
+          "logprob": -13.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.0234375,
+          "logprob": -12.4375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1796875,
+          "logprob": -10.765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6875,
+          "logprob": -13.109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.9453125,
+          "logprob": -12.8515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6875,
+          "logprob": -14.7421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.78125,
+          "logprob": -12.84375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.9609375,
+          "logprob": -12.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6484375,
           "text": "<image>"
         },
         {
@@ -3577,27 +3517,87 @@
         },
         {
           "id": 32000,
-          "logprob": -16.140625,
+          "logprob": -13.4296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4609375,
+          "logprob": -13.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.7265625,
+          "logprob": -12.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.609375,
+          "logprob": -11.5234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.234375,
+          "logprob": -15.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.25,
           "text": "<image>"
         },
         {
@@ -3607,7 +3607,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.4921875,
+          "logprob": -13.5,
           "text": "<image>"
         },
         {
@@ -3632,12 +3632,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.15625,
+          "logprob": -13.1484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -17.046875,
+          "logprob": -17.0625,
           "text": "<image>"
         },
         {
@@ -3647,12 +3647,12 @@
         },
         {
           "id": 32000,
-          "logprob": -14.265625,
+          "logprob": -14.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6328125,
+          "logprob": -12.625,
           "text": "<image>"
         },
         {
@@ -3662,92 +3662,37 @@
         },
         {
           "id": 32000,
-          "logprob": -13.9375,
+          "logprob": -13.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0390625,
+          "logprob": -13.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.46875,
+          "logprob": -10.5,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1640625,
+          "logprob": -13.1953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.59375,
+          "logprob": -13.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.390625,
+          "logprob": -13.3828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8125,
+          "logprob": -14.0625,
           "text": "<image>"
         },
         {
@@ -3755,6 +3700,61 @@
           "logprob": -13.5234375,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -14.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.53125,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -11.4140625,
@@ -3762,17 +3762,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.46875,
+          "logprob": -11.4609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.09375,
+          "logprob": -13.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.34375,
+          "logprob": -12.3359375,
           "text": "<image>"
         },
         {
@@ -3782,32 +3782,32 @@
         },
         {
           "id": 32000,
-          "logprob": -12.7578125,
+          "logprob": -12.75,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.8359375,
+          "logprob": -14.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0859375,
+          "logprob": -13.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1640625,
+          "logprob": -14.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0625,
+          "logprob": -13.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.5078125,
+          "logprob": -14.4921875,
           "text": "<image>"
         },
         {
@@ -3817,7 +3817,7 @@
         },
         {
           "id": 32000,
-          "logprob": -14.6953125,
+          "logprob": -14.703125,
           "text": "<image>"
         },
         {
@@ -3827,32 +3827,32 @@
         },
         {
           "id": 32000,
-          "logprob": -16.328125,
+          "logprob": -16.3125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.9921875,
+          "logprob": -14.9765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.8515625,
+          "logprob": -13.8046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.40625,
+          "logprob": -15.375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4296875,
+          "logprob": -13.3984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.5859375,
+          "logprob": -14.578125,
           "text": "<image>"
         },
         {
@@ -3867,7 +3867,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.3359375,
+          "logprob": -13.328125,
           "text": "<image>"
         },
         {
@@ -3882,37 +3882,52 @@
         },
         {
           "id": 32000,
-          "logprob": -16.890625,
+          "logprob": -16.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.328125,
+          "logprob": -13.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.78125,
+          "logprob": -13.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.34375,
+          "logprob": -13.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4921875,
+          "logprob": -13.4765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4296875,
+          "logprob": -13.390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.28125,
+          "logprob": -13.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5,
           "text": "<image>"
         },
         {
@@ -3922,7 +3937,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.75,
+          "logprob": -13.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6796875,
           "text": "<image>"
         },
         {
@@ -3932,32 +3957,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.328125,
+          "logprob": -13.3203125,
           "text": "<image>"
         },
         {
@@ -3967,47 +3967,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.640625,
+          "logprob": -13.6171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.15625,
+          "logprob": -15.140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0078125,
+          "logprob": -13.21875,
           "text": "<image>"
         },
         {
@@ -4017,27 +3987,57 @@
         },
         {
           "id": 32000,
-          "logprob": -11.625,
+          "logprob": -12.0234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8671875,
+          "logprob": -11.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.578125,
+          "logprob": -11.3125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8125,
+          "logprob": -10.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.9453125,
+          "logprob": -13.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.9765625,
           "text": "<image>"
         },
         {
@@ -4047,17 +4047,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.9765625,
+          "logprob": -12.984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.828125,
+          "logprob": -13.8359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7265625,
+          "logprob": -12.75,
           "text": "<image>"
         },
         {
@@ -4072,12 +4072,12 @@
         },
         {
           "id": 32000,
-          "logprob": -14.7578125,
+          "logprob": -14.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.5234375,
+          "logprob": -14.5078125,
           "text": "<image>"
         },
         {
@@ -4087,12 +4087,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.0390625,
+          "logprob": -13.046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4453125,
+          "logprob": -13.453125,
           "text": "<image>"
         },
         {
@@ -4107,17 +4107,17 @@
         },
         {
           "id": 32000,
-          "logprob": -14.0,
+          "logprob": -13.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5390625,
+          "logprob": -13.4921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0859375,
+          "logprob": -11.9765625,
           "text": "<image>"
         },
         {
@@ -4127,17 +4127,37 @@
         },
         {
           "id": 32000,
-          "logprob": -15.953125,
+          "logprob": -15.9609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.2265625,
+          "logprob": -14.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1484375,
+          "logprob": -13.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.640625,
           "text": "<image>"
         },
         {
@@ -4147,32 +4167,47 @@
         },
         {
           "id": 32000,
-          "logprob": -12.6875,
+          "logprob": -13.625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.796875,
+          "logprob": -12.890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.65625,
+          "logprob": -12.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6796875,
+          "logprob": -12.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.65625,
+          "logprob": -12.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.90625,
+          "logprob": -14.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0078125,
           "text": "<image>"
         },
         {
@@ -4182,52 +4217,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.6171875,
+          "logprob": -15.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7890625,
+          "logprob": -12.921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.375,
+          "logprob": -12.40625,
           "text": "<image>"
         },
         {
@@ -4242,27 +4242,27 @@
         },
         {
           "id": 32000,
-          "logprob": -11.375,
+          "logprob": -11.3671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0625,
+          "logprob": -11.0703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3046875,
+          "logprob": -11.28125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0390625,
+          "logprob": -11.046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3515625,
+          "logprob": -13.3203125,
           "text": "<image>"
         },
         {
@@ -4287,12 +4287,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.8359375,
+          "logprob": -13.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0234375,
+          "logprob": -12.0859375,
           "text": "<image>"
         },
         {
@@ -4302,12 +4302,7 @@
         },
         {
           "id": 32000,
-          "logprob": -14.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0546875,
+          "logprob": -14.0703125,
           "text": "<image>"
         },
         {
@@ -4317,7 +4312,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.0625,
+          "logprob": -14.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0390625,
           "text": "<image>"
         },
         {
@@ -4327,42 +4327,42 @@
         },
         {
           "id": 32000,
-          "logprob": -14.46875,
+          "logprob": -14.453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.703125,
+          "logprob": -14.6875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.296875,
+          "logprob": -14.3046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.8828125,
+          "logprob": -14.8671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1796875,
+          "logprob": -14.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2265625,
+          "logprob": -13.203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1171875,
+          "logprob": -13.109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9375,
+          "logprob": -11.8515625,
           "text": "<image>"
         },
         {
@@ -4377,12 +4377,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.71875,
+          "logprob": -12.7109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.734375,
+          "logprob": -13.7109375,
           "text": "<image>"
         },
         {
@@ -4392,32 +4392,32 @@
         },
         {
           "id": 32000,
-          "logprob": -12.46875,
+          "logprob": -12.4765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.15625,
+          "logprob": -12.140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.375,
+          "logprob": -12.3828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8984375,
+          "logprob": -12.875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.6953125,
+          "logprob": -13.6484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0625,
+          "logprob": -13.0390625,
           "text": "<image>"
         },
         {
@@ -4427,12 +4427,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.6796875,
+          "logprob": -12.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3515625,
+          "logprob": -12.34375,
           "text": "<image>"
         },
         {
@@ -4442,37 +4442,37 @@
         },
         {
           "id": 32000,
-          "logprob": -12.46875,
+          "logprob": -12.453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.03125,
+          "logprob": -12.015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.96875,
+          "logprob": -11.9765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.4609375,
+          "logprob": -14.46875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.328125,
+          "logprob": -13.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6484375,
+          "logprob": -12.71875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.65625,
+          "logprob": -10.6484375,
           "text": "<image>"
         },
         {
@@ -4492,22 +4492,22 @@
         },
         {
           "id": 32000,
-          "logprob": -12.6328125,
+          "logprob": -12.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.265625,
+          "logprob": -11.2578125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.734375,
+          "logprob": -11.7265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.84375,
+          "logprob": -10.8515625,
           "text": "<image>"
         },
         {
@@ -4517,32 +4517,32 @@
         },
         {
           "id": 32000,
-          "logprob": -13.109375,
+          "logprob": -13.140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7421875,
+          "logprob": -12.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.09375,
+          "logprob": -12.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.109375,
+          "logprob": -13.1328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.671875,
+          "logprob": -12.65625,
           "text": "<image>"
         },
         {
@@ -4552,27 +4552,27 @@
         },
         {
           "id": 32000,
-          "logprob": -12.6796875,
+          "logprob": -12.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.109375,
+          "logprob": -12.0859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.625,
+          "logprob": -13.609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8515625,
+          "logprob": -12.8359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3046875,
+          "logprob": -12.3125,
           "text": "<image>"
         },
         {
@@ -4582,12 +4582,12 @@
         },
         {
           "id": 32000,
-          "logprob": -14.0703125,
+          "logprob": -14.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.0,
+          "logprob": -13.984375,
           "text": "<image>"
         },
         {
@@ -4597,7 +4597,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.2421875,
+          "logprob": -13.2109375,
           "text": "<image>"
         },
         {
@@ -4607,42 +4607,42 @@
         },
         {
           "id": 32000,
-          "logprob": -13.7109375,
+          "logprob": -13.703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6015625,
+          "logprob": -12.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.796875,
+          "logprob": -15.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.90625,
+          "logprob": -11.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.375,
+          "logprob": -12.3671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4765625,
+          "logprob": -11.46875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.015625,
+          "logprob": -12.0,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.328125,
+          "logprob": -11.3359375,
           "text": "<image>"
         },
         {
@@ -4657,17 +4657,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.8203125,
+          "logprob": -13.828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7578125,
+          "logprob": -11.75,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.484375,
+          "logprob": -11.46875,
           "text": "<image>"
         },
         {
@@ -4682,17 +4682,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.640625,
+          "logprob": -11.625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5859375,
+          "logprob": -11.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.2578125,
+          "logprob": -15.2265625,
           "text": "<image>"
         },
         {
@@ -4702,17 +4702,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.0078125,
+          "logprob": -13.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.3828125,
+          "logprob": -15.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.203125,
+          "logprob": -11.1875,
           "text": "<image>"
         },
         {
@@ -4722,7 +4722,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.0703125,
+          "logprob": -11.0625,
           "text": "<image>"
         },
         {
@@ -4732,7 +4732,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.4453125,
+          "logprob": -13.453125,
           "text": "<image>"
         },
         {
@@ -4742,27 +4742,27 @@
         },
         {
           "id": 32000,
-          "logprob": -11.5078125,
+          "logprob": -11.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.015625,
+          "logprob": -12.0078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.15625,
+          "logprob": -11.1875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8359375,
+          "logprob": -11.890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.359375,
+          "logprob": -11.3125,
           "text": "<image>"
         },
         {
@@ -4777,22 +4777,22 @@
         },
         {
           "id": 32000,
-          "logprob": -12.4453125,
+          "logprob": -12.4921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.015625,
+          "logprob": -12.0703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.375,
+          "logprob": -13.3671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2734375,
+          "logprob": -13.28125,
           "text": "<image>"
         },
         {
@@ -4802,22 +4802,22 @@
         },
         {
           "id": 32000,
-          "logprob": -13.046875,
+          "logprob": -13.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0859375,
+          "logprob": -13.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1640625,
+          "logprob": -12.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4765625,
+          "logprob": -13.484375,
           "text": "<image>"
         },
         {
@@ -4827,7 +4827,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.7265625,
+          "logprob": -13.703125,
           "text": "<image>"
         },
         {
@@ -4837,262 +4837,17 @@
         },
         {
           "id": 32000,
-          "logprob": -14.3359375,
+          "logprob": -14.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.71875,
+          "logprob": -12.7109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.640625,
+          "logprob": -14.28125,
           "text": "<image>"
         },
         {
@@ -5102,52 +4857,57 @@
         },
         {
           "id": 32000,
-          "logprob": -13.453125,
+          "logprob": -13.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.4375,
+          "logprob": -15.203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1171875,
+          "logprob": -13.28125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4296875,
+          "logprob": -12.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.515625,
+          "logprob": -14.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.21875,
+          "logprob": -13.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8671875,
+          "logprob": -12.1171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8515625,
+          "logprob": -14.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.8671875,
+          "logprob": -14.0625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.59375,
+          "logprob": -12.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.953125,
           "text": "<image>"
         },
         {
@@ -5157,12 +4917,57 @@
         },
         {
           "id": 32000,
-          "logprob": -12.1640625,
+          "logprob": -11.6953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2265625,
+          "logprob": -12.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5234375,
           "text": "<image>"
         },
         {
@@ -5172,32 +4977,27 @@
         },
         {
           "id": 32000,
-          "logprob": -12.1015625,
+          "logprob": -11.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0234375,
+          "logprob": -11.2578125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4140625,
+          "logprob": -13.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.828125,
+          "logprob": -11.921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1953125,
+          "logprob": -13.03125,
           "text": "<image>"
         },
         {
@@ -5207,7 +5007,207 @@
         },
         {
           "id": 32000,
-          "logprob": -11.1328125,
+          "logprob": -14.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.4296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.140625,
           "text": "<image>"
         },
         {
@@ -5232,7 +5232,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.1015625,
+          "logprob": -11.1171875,
           "text": "<image>"
         },
         {
@@ -5242,7 +5242,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.15625,
+          "logprob": -11.1484375,
           "text": "<image>"
         },
         {
@@ -5252,17 +5252,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.4296875,
+          "logprob": -11.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.765625,
+          "logprob": -12.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.09375,
+          "logprob": -13.0859375,
           "text": "<image>"
         },
         {
@@ -5272,22 +5272,22 @@
         },
         {
           "id": 32000,
-          "logprob": -12.96875,
+          "logprob": -12.9609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.328125,
+          "logprob": -12.34375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8359375,
+          "logprob": -12.84375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.9609375,
+          "logprob": -13.9765625,
           "text": "<image>"
         },
         {
@@ -5297,7 +5297,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.46875,
+          "logprob": -13.484375,
           "text": "<image>"
         },
         {
@@ -5307,22 +5307,22 @@
         },
         {
           "id": 32000,
-          "logprob": -13.59375,
+          "logprob": -13.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.40625,
+          "logprob": -13.3828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.234375,
+          "logprob": -14.25,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.265625,
+          "logprob": -14.2578125,
           "text": "<image>"
         },
         {
@@ -5332,127 +5332,127 @@
         },
         {
           "id": 32000,
-          "logprob": -13.65625,
+          "logprob": -13.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.2734375,
+          "logprob": -14.2578125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.09375,
+          "logprob": -13.8671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.890625,
+          "logprob": -12.921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.640625,
+          "logprob": -15.6484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0234375,
+          "logprob": -13.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3828125,
+          "logprob": -12.34375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.71875,
+          "logprob": -12.7265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.5234375,
+          "logprob": -14.515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7734375,
+          "logprob": -12.75,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3203125,
+          "logprob": -12.265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.609375,
+          "logprob": -13.546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1640625,
+          "logprob": -13.1328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.203125,
+          "logprob": -12.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1875,
+          "logprob": -12.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.59375,
+          "logprob": -11.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6875,
+          "logprob": -11.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1796875,
+          "logprob": -12.15625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6484375,
+          "logprob": -11.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.421875,
+          "logprob": -11.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.34375,
+          "logprob": -11.390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1015625,
+          "logprob": -11.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9765625,
+          "logprob": -11.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.015625,
+          "logprob": -12.0078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0,
+          "logprob": -12.984375,
           "text": "<image>"
         },
         {
@@ -5462,7 +5462,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.09375,
           "text": "<image>"
         },
         {
@@ -5482,12 +5482,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.65625,
+          "logprob": -11.6484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.84375,
+          "logprob": -12.8125,
           "text": "<image>"
         },
         {
@@ -5497,17 +5497,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.6953125,
+          "logprob": -11.6796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.578125,
+          "logprob": -11.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.6796875,
+          "logprob": -13.6953125,
           "text": "<image>"
         },
         {
@@ -5522,7 +5522,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0078125,
+          "logprob": -12.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.296875,
           "text": "<image>"
         },
         {
@@ -5532,17 +5537,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.234375,
+          "logprob": -13.84375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8671875,
+          "logprob": -12.875,
           "text": "<image>"
         },
         {
@@ -5552,17 +5552,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.5078125,
+          "logprob": -13.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.953125,
+          "logprob": -13.9609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5078125,
+          "logprob": -12.46875,
           "text": "<image>"
         },
         {
@@ -5572,232 +5572,17 @@
         },
         {
           "id": 32000,
-          "logprob": -14.25,
+          "logprob": -14.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9140625,
+          "logprob": -12.921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4375,
+          "logprob": -14.2578125,
           "text": "<image>"
         },
         {
@@ -5807,62 +5592,62 @@
         },
         {
           "id": 32000,
-          "logprob": -13.78125,
+          "logprob": -14.3984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5703125,
+          "logprob": -15.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9765625,
+          "logprob": -13.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4765625,
+          "logprob": -12.3515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.125,
+          "logprob": -13.46875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0859375,
+          "logprob": -11.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9296875,
+          "logprob": -12.2890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.90625,
+          "logprob": -14.3515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0625,
+          "logprob": -13.140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0234375,
+          "logprob": -13.0234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.6953125,
+          "logprob": -12.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.890625,
+          "logprob": -11.8125,
           "text": "<image>"
         },
         {
@@ -5870,6 +5655,31 @@
           "logprob": -12.125,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -11.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.640625,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -12.4375,
@@ -5877,67 +5687,257 @@
         },
         {
           "id": 32000,
-          "logprob": -12.375,
+          "logprob": -11.46875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.28125,
+          "logprob": -11.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4921875,
+          "logprob": -12.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1640625,
+          "logprob": -11.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1953125,
+          "logprob": -11.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.4140625,
+          "logprob": -12.4453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6171875,
+          "logprob": -11.34375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.921875,
+          "logprob": -12.1015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6328125,
+          "logprob": -11.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2421875,
+          "logprob": -14.6015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0234375,
+          "logprob": -12.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6953125,
+          "logprob": -11.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1484375,
+          "logprob": -11.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1328125,
           "text": "<image>"
         },
         {
@@ -5947,27 +5947,27 @@
         },
         {
           "id": 32000,
-          "logprob": -11.7265625,
+          "logprob": -11.71875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6796875,
+          "logprob": -11.6875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4921875,
+          "logprob": -12.484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.875,
+          "logprob": -11.8828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5390625,
+          "logprob": -12.53125,
           "text": "<image>"
         },
         {
@@ -5982,27 +5982,27 @@
         },
         {
           "id": 32000,
-          "logprob": -13.71875,
+          "logprob": -13.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.78125,
+          "logprob": -14.8046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.578125,
+          "logprob": -11.609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5859375,
+          "logprob": -13.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.390625,
+          "logprob": -12.421875,
           "text": "<image>"
         },
         {
@@ -6012,17 +6012,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.65625,
+          "logprob": -13.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5,
+          "logprob": -12.5078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.109375,
+          "logprob": -13.125,
           "text": "<image>"
         },
         {
@@ -6032,37 +6032,37 @@
         },
         {
           "id": 32000,
-          "logprob": -12.2578125,
+          "logprob": -12.2890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7421875,
+          "logprob": -12.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1640625,
+          "logprob": -13.15625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9375,
+          "logprob": -11.9453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.265625,
+          "logprob": -12.28125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1640625,
+          "logprob": -12.1328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.8671875,
+          "logprob": -13.8203125,
           "text": "<image>"
         },
         {
@@ -6072,57 +6072,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.046875,
+          "logprob": -13.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.984375,
+          "logprob": -15.6953125,
           "text": "<image>"
         },
         {
@@ -6132,12 +6087,47 @@
         },
         {
           "id": 32000,
-          "logprob": -11.3984375,
+          "logprob": -14.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.796875,
+          "logprob": -13.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.234375,
           "text": "<image>"
         },
         {
@@ -6147,37 +6137,47 @@
         },
         {
           "id": 32000,
-          "logprob": -11.4296875,
+          "logprob": -11.828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9921875,
+          "logprob": -11.453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.59375,
+          "logprob": -11.453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8359375,
+          "logprob": -11.984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7421875,
+          "logprob": -11.609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.125,
+          "logprob": -11.8046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.875,
+          "logprob": -11.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8515625,
           "text": "<image>"
         },
         {
@@ -6187,27 +6187,27 @@
         },
         {
           "id": 32000,
-          "logprob": -10.796875,
+          "logprob": -10.8046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4765625,
+          "logprob": -11.578125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1484375,
+          "logprob": -11.15625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8046875,
+          "logprob": -12.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3828125,
+          "logprob": -11.3671875,
           "text": "<image>"
         },
         {
@@ -6227,7 +6227,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.5546875,
+          "logprob": -12.5625,
           "text": "<image>"
         },
         {
@@ -6237,367 +6237,57 @@
         },
         {
           "id": 32000,
-          "logprob": -10.40625,
+          "logprob": -10.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.59375,
+          "logprob": -12.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.03125,
+          "logprob": -12.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.03125,
+          "logprob": -12.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.65625,
+          "logprob": -12.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6328125,
+          "logprob": -11.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5,
+          "logprob": -13.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.078125,
+          "logprob": -12.0703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0234375,
+          "logprob": -13.015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.171875,
+          "logprob": -14.1484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.4609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.015625,
+          "logprob": -12.3671875,
           "text": "<image>"
         },
         {
@@ -6607,42 +6297,112 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0859375,
+          "logprob": -12.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.140625,
+          "logprob": -13.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.65625,
+          "logprob": -13.953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.890625,
+          "logprob": -12.4765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.59375,
+          "logprob": -14.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4375,
+          "logprob": -12.6015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4375,
+          "logprob": -13.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.109375,
+          "logprob": -15.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0546875,
           "text": "<image>"
         },
         {
@@ -6652,17 +6412,257 @@
         },
         {
           "id": 32000,
-          "logprob": -11.1015625,
+          "logprob": -11.9609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.203125,
+          "logprob": -14.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9921875,
+          "logprob": -12.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.1484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.96875,
           "text": "<image>"
         },
         {
@@ -6672,32 +6672,32 @@
         },
         {
           "id": 32000,
-          "logprob": -11.40625,
+          "logprob": -11.4296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.53125,
+          "logprob": -12.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.109375,
+          "logprob": -12.1015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3203125,
+          "logprob": -11.3125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.125,
+          "logprob": -11.1171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7578125,
+          "logprob": -11.75,
           "text": "<image>"
         },
         {
@@ -6712,7 +6712,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.109375,
+          "logprob": -11.1015625,
           "text": "<image>"
         },
         {
@@ -6732,67 +6732,67 @@
         },
         {
           "id": 32000,
-          "logprob": -12.609375,
+          "logprob": -12.6015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.65625,
+          "logprob": -12.6796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7890625,
+          "logprob": -12.8359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.28125,
+          "logprob": -12.296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4140625,
+          "logprob": -13.3671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.96875,
+          "logprob": -12.9765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1171875,
+          "logprob": -12.109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5390625,
+          "logprob": -12.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.609375,
+          "logprob": -12.6015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6484375,
+          "logprob": -12.625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.09375,
+          "logprob": -12.109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9921875,
+          "logprob": -13.0,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4453125,
+          "logprob": -13.4296875,
           "text": "<image>"
         },
         {
@@ -6802,17 +6802,17 @@
         },
         {
           "id": 32000,
-          "logprob": -14.7109375,
+          "logprob": -14.71875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.625,
+          "logprob": -13.6015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.765625,
+          "logprob": -13.7578125,
           "text": "<image>"
         },
         {
@@ -6822,37 +6822,37 @@
         },
         {
           "id": 32000,
-          "logprob": -14.53125,
+          "logprob": -14.5390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.34375,
+          "logprob": -13.3515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.921875,
+          "logprob": -12.9453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.359375,
+          "logprob": -11.3515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.078125,
+          "logprob": -12.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4609375,
+          "logprob": -11.421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.640625,
+          "logprob": -11.625,
           "text": "<image>"
         },
         {
@@ -6867,62 +6867,62 @@
         },
         {
           "id": 32000,
-          "logprob": -11.953125,
+          "logprob": -11.9453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0859375,
+          "logprob": -12.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.421875,
+          "logprob": -11.4296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5859375,
+          "logprob": -11.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0859375,
+          "logprob": -12.0703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.25,
+          "logprob": -11.2421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6171875,
+          "logprob": -11.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3359375,
+          "logprob": -11.3125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.46875,
+          "logprob": -11.453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.265625,
+          "logprob": -12.28125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.21875,
+          "logprob": -12.2109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1875,
+          "logprob": -12.203125,
           "text": "<image>"
         },
         {
@@ -6937,47 +6937,47 @@
         },
         {
           "id": 32000,
-          "logprob": -11.2421875,
+          "logprob": -11.25,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3046875,
+          "logprob": -12.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.03125,
+          "logprob": -11.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.203125,
+          "logprob": -11.1953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.125,
+          "logprob": -11.1171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9296875,
+          "logprob": -11.9375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9765625,
+          "logprob": -10.984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.453125,
+          "logprob": -11.390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.859375,
+          "logprob": -12.8828125,
           "text": "<image>"
         },
         {
@@ -7002,47 +7002,47 @@
         },
         {
           "id": 32000,
-          "logprob": -12.1328125,
+          "logprob": -12.1171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5,
+          "logprob": -12.515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8671875,
+          "logprob": -11.890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1640625,
+          "logprob": -13.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.765625,
+          "logprob": -15.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.609375,
+          "logprob": -11.6015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.34375,
+          "logprob": -13.328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.671875,
+          "logprob": -13.6484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.9140625,
+          "logprob": -14.875,
           "text": "<image>"
         },
         {
@@ -7052,57 +7052,87 @@
         },
         {
           "id": 32000,
-          "logprob": -13.53125,
+          "logprob": -13.5234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.25,
+          "logprob": -13.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6171875,
+          "logprob": -12.609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.8046875,
+          "logprob": -13.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.375,
+          "logprob": -14.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.171875,
+          "logprob": -12.1484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.828125,
+          "logprob": -11.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.875,
+          "logprob": -11.890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.484375,
+          "logprob": -11.4921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5625,
+          "logprob": -12.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.640625,
+          "logprob": -11.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.03125,
           "text": "<image>"
         },
         {
@@ -7110,36 +7140,6 @@
           "logprob": -11.7265625,
           "text": "<image>"
         },
-        {
-          "id": 32000,
-          "logprob": -11.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.734375,
-          "text": "<image>"
-        },
         {
           "id": 32000,
           "logprob": -11.84375,
@@ -7147,12 +7147,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0390625,
+          "logprob": -11.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.2578125,
+          "logprob": -11.2421875,
           "text": "<image>"
         },
         {
@@ -7187,22 +7187,22 @@
         },
         {
           "id": 32000,
-          "logprob": -11.0390625,
+          "logprob": -11.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1953125,
+          "logprob": -12.203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.6640625,
+          "logprob": -13.609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6640625,
+          "logprob": -12.65625,
           "text": "<image>"
         },
         {
@@ -7227,27 +7227,47 @@
         },
         {
           "id": 32000,
-          "logprob": -11.9140625,
+          "logprob": -11.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.34375,
+          "logprob": -11.3359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.578125,
+          "logprob": -11.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.953125,
+          "logprob": -11.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.296875,
+          "logprob": -11.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.875,
           "text": "<image>"
         },
         {
@@ -7257,27 +7277,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9765625,
+          "logprob": -12.046875,
           "text": "<image>"
         },
         {
@@ -7287,7 +7287,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.734375,
+          "logprob": -13.7265625,
           "text": "<image>"
         },
         {
@@ -7297,7 +7297,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.8359375,
+          "logprob": -13.8203125,
           "text": "<image>"
         },
         {
@@ -7312,37 +7312,37 @@
         },
         {
           "id": 32000,
-          "logprob": -14.5625,
+          "logprob": -14.546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.0234375,
+          "logprob": -14.015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5703125,
+          "logprob": -12.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8125,
+          "logprob": -11.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.7109375,
+          "logprob": -10.6953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6015625,
+          "logprob": -11.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8671875,
+          "logprob": -12.0234375,
           "text": "<image>"
         },
         {
@@ -7357,32 +7357,32 @@
         },
         {
           "id": 32000,
-          "logprob": -14.3359375,
+          "logprob": -14.34375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.90625,
+          "logprob": -11.8984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.296875,
+          "logprob": -13.4375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.984375,
+          "logprob": -11.9765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0625,
+          "logprob": -11.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4296875,
+          "logprob": -11.421875,
           "text": "<image>"
         },
         {
@@ -7397,27 +7397,27 @@
         },
         {
           "id": 32000,
-          "logprob": -11.28125,
+          "logprob": -11.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5546875,
+          "logprob": -11.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.671875,
+          "logprob": -11.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1171875,
+          "logprob": -12.125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5,
+          "logprob": -11.484375,
           "text": "<image>"
         },
         {
@@ -7427,27 +7427,27 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8984375,
+          "logprob": -11.875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2890625,
+          "logprob": -12.3125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6484375,
+          "logprob": -11.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8984375,
+          "logprob": -12.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.734375,
+          "logprob": -11.7421875,
           "text": "<image>"
         },
         {
@@ -7457,7 +7457,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.40625,
+          "logprob": -11.3984375,
           "text": "<image>"
         },
         {
@@ -7467,7 +7467,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.3515625,
+          "logprob": -11.34375,
           "text": "<image>"
         },
         {
@@ -7482,37 +7482,37 @@
         },
         {
           "id": 32000,
-          "logprob": -11.03125,
+          "logprob": -10.9609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.375,
+          "logprob": -13.265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6640625,
+          "logprob": -12.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.96875,
+          "logprob": -11.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.671875,
+          "logprob": -14.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.390625,
+          "logprob": -12.3828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.2890625,
+          "logprob": -14.3515625,
           "text": "<image>"
         },
         {
@@ -7522,32 +7522,32 @@
         },
         {
           "id": 32000,
-          "logprob": -15.1796875,
+          "logprob": -15.125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6875,
+          "logprob": -12.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7421875,
+          "logprob": -12.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.6484375,
+          "logprob": -14.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.546875,
+          "logprob": -13.5234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5703125,
+          "logprob": -12.5859375,
           "text": "<image>"
         },
         {
@@ -7557,32 +7557,17 @@
         },
         {
           "id": 32000,
-          "logprob": -14.390625,
+          "logprob": -14.3828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.96875,
+          "logprob": -12.9453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7421875,
+          "logprob": -12.609375,
           "text": "<image>"
         },
         {
@@ -7592,32 +7577,47 @@
         },
         {
           "id": 32000,
-          "logprob": -12.453125,
+          "logprob": -12.953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.8046875,
+          "logprob": -11.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.0078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.7890625,
+          "logprob": -12.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.3828125,
+          "logprob": -13.8828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.828125,
+          "logprob": -12.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8359375,
           "text": "<image>"
         },
         {
@@ -7627,12 +7627,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.0,
+          "logprob": -13.0078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1171875,
+          "logprob": -12.140625,
           "text": "<image>"
         },
         {
@@ -7642,17 +7642,17 @@
         },
         {
           "id": 32000,
-          "logprob": -14.015625,
+          "logprob": -13.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6328125,
+          "logprob": -11.625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4765625,
+          "logprob": -11.484375,
           "text": "<image>"
         },
         {
@@ -7662,7 +7662,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.5859375,
+          "logprob": -11.5703125,
           "text": "<image>"
         },
         {
@@ -7677,7 +7677,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0390625,
+          "logprob": -12.03125,
           "text": "<image>"
         },
         {
@@ -7687,7 +7687,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.4609375,
+          "logprob": -11.46875,
           "text": "<image>"
         },
         {
@@ -7697,7 +7697,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0625,
+          "logprob": -12.078125,
           "text": "<image>"
         },
         {
@@ -7707,142 +7707,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
+          "logprob": -11.296875,
           "text": "<image>"
         },
         {
@@ -7852,7 +7717,142 @@
         },
         {
           "id": 32000,
-          "logprob": -11.34375,
+          "logprob": -11.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.21875,
           "text": "<image>"
         },
         {
@@ -7862,52 +7862,52 @@
         },
         {
           "id": 32000,
-          "logprob": -11.3671875,
+          "logprob": -11.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0,
+          "logprob": -12.984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.1875,
+          "logprob": -15.15625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0,
+          "logprob": -11.9765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.75,
+          "logprob": -12.765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.515625,
+          "logprob": -13.4921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5703125,
+          "logprob": -11.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.703125,
+          "logprob": -11.6953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3671875,
+          "logprob": -11.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4921875,
+          "logprob": -11.5,
           "text": "<image>"
         },
         {
@@ -7917,12 +7917,12 @@
         },
         {
           "id": 32000,
-          "logprob": -14.5546875,
+          "logprob": -14.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.390625,
+          "logprob": -13.4140625,
           "text": "<image>"
         },
         {
@@ -7932,7 +7932,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.7265625,
+          "logprob": -11.71875,
           "text": "<image>"
         },
         {
@@ -7947,7 +7947,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.2421875,
+          "logprob": -13.2578125,
           "text": "<image>"
         },
         {
@@ -7967,27 +7967,27 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8203125,
+          "logprob": -11.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9296875,
+          "logprob": -11.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.796875,
+          "logprob": -11.765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1875,
+          "logprob": -11.1953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.359375,
+          "logprob": -13.3359375,
           "text": "<image>"
         },
         {
@@ -7997,7 +7997,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.6796875,
+          "logprob": -12.703125,
           "text": "<image>"
         },
         {
@@ -8012,27 +8012,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.5625,
+          "logprob": -13.6171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6484375,
+          "logprob": -13.328125,
           "text": "<image>"
         },
         {
@@ -8040,6 +8025,21 @@
           "logprob": -12.828125,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -13.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.796875,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -12.78125,
@@ -8047,42 +8047,42 @@
         },
         {
           "id": 32000,
-          "logprob": -12.578125,
+          "logprob": -12.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7578125,
+          "logprob": -12.765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.484375,
+          "logprob": -11.46875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5390625,
+          "logprob": -12.484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6015625,
+          "logprob": -11.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.640625,
+          "logprob": -11.6171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.140625,
+          "logprob": -12.1328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.65625,
+          "logprob": -12.59375,
           "text": "<image>"
         },
         {
@@ -8092,17 +8092,17 @@
         },
         {
           "id": 32000,
-          "logprob": -14.265625,
+          "logprob": -14.25,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0078125,
+          "logprob": -12.0234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7578125,
+          "logprob": -11.7734375,
           "text": "<image>"
         },
         {
@@ -8117,52 +8117,52 @@
         },
         {
           "id": 32000,
-          "logprob": -11.625,
+          "logprob": -11.609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.53125,
+          "logprob": -13.5234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.046875,
+          "logprob": -12.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6796875,
+          "logprob": -12.6953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7109375,
+          "logprob": -11.703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.890625,
+          "logprob": -11.8828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8359375,
+          "logprob": -11.84375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7734375,
+          "logprob": -11.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3671875,
+          "logprob": -12.3515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.3125,
+          "logprob": -14.3046875,
           "text": "<image>"
         },
         {
@@ -8172,22 +8172,22 @@
         },
         {
           "id": 32000,
-          "logprob": -11.921875,
+          "logprob": -11.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3203125,
+          "logprob": -11.328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1640625,
+          "logprob": -14.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.2421875,
+          "logprob": -11.25,
           "text": "<image>"
         },
         {
@@ -8197,7 +8197,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.078125,
           "text": "<image>"
         },
         {
@@ -8207,12 +8207,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.2578125,
+          "logprob": -12.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.7265625,
+          "logprob": -13.71875,
           "text": "<image>"
         },
         {
@@ -8232,12 +8232,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.3828125,
+          "logprob": -13.390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5390625,
+          "logprob": -12.515625,
           "text": "<image>"
         },
         {
@@ -8247,22 +8247,22 @@
         },
         {
           "id": 32000,
-          "logprob": -14.15625,
+          "logprob": -14.1796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5703125,
+          "logprob": -11.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5546875,
+          "logprob": -11.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.65625,
+          "logprob": -11.671875,
           "text": "<image>"
         },
         {
@@ -8272,47 +8272,47 @@
         },
         {
           "id": 32000,
-          "logprob": -12.6484375,
+          "logprob": -12.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5390625,
+          "logprob": -12.515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7734375,
+          "logprob": -11.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0859375,
+          "logprob": -13.1328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9140625,
+          "logprob": -12.8984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.6015625,
+          "logprob": -14.609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5390625,
+          "logprob": -11.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8828125,
+          "logprob": -11.8515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.078125,
+          "logprob": -13.0625,
           "text": "<image>"
         },
         {
@@ -8322,17 +8322,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.15625,
+          "logprob": -12.1328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.03125,
+          "logprob": -12.0234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4453125,
+          "logprob": -12.453125,
           "text": "<image>"
         },
         {
@@ -8342,7 +8342,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.9375,
+          "logprob": -11.9296875,
           "text": "<image>"
         },
         {
@@ -8357,42 +8357,42 @@
         },
         {
           "id": 32000,
-          "logprob": -12.7890625,
+          "logprob": -12.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0390625,
+          "logprob": -13.0234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8203125,
+          "logprob": -12.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.578125,
+          "logprob": -12.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.7578125,
+          "logprob": -10.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9609375,
+          "logprob": -11.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.484375,
+          "logprob": -11.4921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.265625,
+          "logprob": -11.25,
           "text": "<image>"
         },
         {
@@ -8402,17 +8402,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.71875,
+          "logprob": -12.6953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7890625,
+          "logprob": -11.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5234375,
+          "logprob": -11.515625,
           "text": "<image>"
         },
         {
@@ -8422,7 +8422,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.46875,
+          "logprob": -12.4765625,
           "text": "<image>"
         },
         {
@@ -8437,17 +8437,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.65625,
+          "logprob": -11.6484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5234375,
+          "logprob": -11.515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7890625,
+          "logprob": -12.8125,
           "text": "<image>"
         },
         {
@@ -8457,17 +8457,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.28125,
+          "logprob": -13.2890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.109375,
+          "logprob": -11.09375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.34375,
+          "logprob": -12.3359375,
           "text": "<image>"
         },
         {
@@ -8477,27 +8477,27 @@
         },
         {
           "id": 32000,
-          "logprob": -13.0,
+          "logprob": -13.0078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.28125,
+          "logprob": -15.328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.8046875,
+          "logprob": -14.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0234375,
+          "logprob": -11.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.109375,
+          "logprob": -13.0703125,
           "text": "<image>"
         },
         {
@@ -8507,12 +8507,12 @@
         },
         {
           "id": 32000,
-          "logprob": -14.0078125,
+          "logprob": -14.0859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.7109375,
+          "logprob": -14.734375,
           "text": "<image>"
         },
         {
@@ -8527,127 +8527,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.9765625,
+          "logprob": -13.953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.3046875,
+          "logprob": -14.28125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.78125,
+          "logprob": -14.8828125,
           "text": "<image>"
         },
         {
@@ -8657,22 +8547,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.859375,
+          "logprob": -12.28125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.765625,
+          "logprob": -12.34375,
           "text": "<image>"
         },
         {
@@ -8687,117 +8567,87 @@
         },
         {
           "id": 32000,
-          "logprob": -11.71875,
+          "logprob": -12.4296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.25,
+          "logprob": -11.8984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.65625,
+          "logprob": -11.984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.84375,
+          "logprob": -11.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0390625,
+          "logprob": -11.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4921875,
+          "logprob": -12.1875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8828125,
+          "logprob": -11.6171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1015625,
+          "logprob": -12.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.0625,
+          "logprob": -12.4765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.09375,
+          "logprob": -12.953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2578125,
+          "logprob": -10.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.4453125,
+          "logprob": -13.1015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9921875,
+          "logprob": -12.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.140625,
+          "logprob": -11.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.828125,
+          "logprob": -11.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5546875,
+          "logprob": -11.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6015625,
+          "logprob": -11.765625,
           "text": "<image>"
         },
         {
@@ -8807,32 +8657,182 @@
         },
         {
           "id": 32000,
-          "logprob": -12.84375,
+          "logprob": -12.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.78125,
+          "logprob": -11.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1171875,
+          "logprob": -12.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8046875,
+          "logprob": -11.71875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5234375,
+          "logprob": -11.34375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4609375,
+          "logprob": -12.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4765625,
           "text": "<image>"
         },
         {
@@ -8852,22 +8852,22 @@
         },
         {
           "id": 32000,
-          "logprob": -12.953125,
+          "logprob": -12.9609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.859375,
+          "logprob": -12.8515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0390625,
+          "logprob": -12.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.28125,
+          "logprob": -13.234375,
           "text": "<image>"
         },
         {
@@ -8877,12 +8877,12 @@
         },
         {
           "id": 32000,
-          "logprob": -10.8203125,
+          "logprob": -10.8515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.6953125,
+          "logprob": -10.671875,
           "text": "<image>"
         },
         {
@@ -8892,7 +8892,7 @@
         },
         {
           "id": 32000,
-          "logprob": -10.9375,
+          "logprob": -10.9140625,
           "text": "<image>"
         },
         {
@@ -8912,17 +8912,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.3046875,
+          "logprob": -12.2890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9609375,
+          "logprob": -10.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6328125,
+          "logprob": -11.6015625,
           "text": "<image>"
         },
         {
@@ -8932,17 +8932,17 @@
         },
         {
           "id": 32000,
-          "logprob": -10.84375,
+          "logprob": -10.8515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.140625,
+          "logprob": -11.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.09375,
+          "logprob": -11.0859375,
           "text": "<image>"
         },
         {
@@ -8952,12 +8952,12 @@
         },
         {
           "id": 32000,
-          "logprob": -15.578125,
+          "logprob": -15.765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.875,
+          "logprob": -11.8515625,
           "text": "<image>"
         },
         {
@@ -8972,22 +8972,22 @@
         },
         {
           "id": 32000,
-          "logprob": -10.984375,
+          "logprob": -10.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.234375,
+          "logprob": -12.25,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8359375,
+          "logprob": -11.828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8515625,
+          "logprob": -11.84375,
           "text": "<image>"
         },
         {
@@ -9002,7 +9002,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.5625,
+          "logprob": -13.5703125,
           "text": "<image>"
         },
         {
@@ -9012,37 +9012,37 @@
         },
         {
           "id": 32000,
-          "logprob": -13.5859375,
+          "logprob": -13.6171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.0234375,
+          "logprob": -14.0078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1796875,
+          "logprob": -11.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1484375,
+          "logprob": -14.140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.234375,
+          "logprob": -12.25,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.734375,
+          "logprob": -11.7265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3515625,
+          "logprob": -12.3203125,
           "text": "<image>"
         },
         {
@@ -9052,17 +9052,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.90625,
+          "logprob": -11.8828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3359375,
+          "logprob": -12.3125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5703125,
+          "logprob": -11.5625,
           "text": "<image>"
         },
         {
@@ -9072,17 +9072,17 @@
         },
         {
           "id": 32000,
-          "logprob": -14.6875,
+          "logprob": -14.6796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0625,
+          "logprob": -11.046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5234375,
+          "logprob": -11.5,
           "text": "<image>"
         },
         {
@@ -9107,12 +9107,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.46875,
+          "logprob": -12.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.0859375,
           "text": "<image>"
         },
         {
@@ -9125,21 +9125,6 @@
           "logprob": -12.46875,
           "text": "<image>"
         },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6640625,
-          "text": "<image>"
-        },
         {
           "id": 32000,
           "logprob": -11.9765625,
@@ -9147,7 +9132,22 @@
         },
         {
           "id": 32000,
-          "logprob": -10.8671875,
+          "logprob": -11.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.875,
           "text": "<image>"
         },
         {
@@ -9167,12 +9167,12 @@
         },
         {
           "id": 32000,
-          "logprob": -10.796875,
+          "logprob": -10.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.59375,
+          "logprob": -11.6171875,
           "text": "<image>"
         },
         {
@@ -9202,22 +9202,22 @@
         },
         {
           "id": 32000,
-          "logprob": -11.390625,
+          "logprob": -11.3828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9375,
+          "logprob": -10.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.6796875,
+          "logprob": -10.703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9375,
+          "logprob": -11.90625,
           "text": "<image>"
         },
         {
@@ -9227,82 +9227,57 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8125,
+          "logprob": -11.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3671875,
+          "logprob": -11.4453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -9.625,
+          "logprob": -9.6328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.8515625,
+          "logprob": -10.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9453125,
+          "logprob": -13.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8203125,
+          "logprob": -12.890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3359375,
+          "logprob": -13.4765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.015625,
+          "logprob": -14.109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8671875,
+          "logprob": -11.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4453125,
+          "logprob": -13.484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.75,
+          "logprob": -13.5625,
           "text": "<image>"
         },
         {
@@ -9310,6 +9285,31 @@
           "logprob": -12.859375,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -10.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.890625,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -13.0625,
@@ -9317,7 +9317,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.21875,
+          "logprob": -11.2265625,
           "text": "<image>"
         },
         {
@@ -9327,7 +9327,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.296875,
+          "logprob": -12.2890625,
           "text": "<image>"
         },
         {
@@ -9337,12 +9337,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.8671875,
+          "logprob": -12.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.6953125,
+          "logprob": -15.671875,
           "text": "<image>"
         },
         {
@@ -9357,7 +9357,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.140625,
+          "logprob": -12.1328125,
           "text": "<image>"
         },
         {
@@ -9367,22 +9367,22 @@
         },
         {
           "id": 32000,
-          "logprob": -11.390625,
+          "logprob": -11.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.328125,
+          "logprob": -12.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.265625,
+          "logprob": -12.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9453125,
+          "logprob": -10.9296875,
           "text": "<image>"
         },
         {
@@ -9397,7 +9397,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.6015625,
+          "logprob": -11.59375,
           "text": "<image>"
         },
         {
@@ -9407,7 +9407,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.546875,
+          "logprob": -11.53125,
           "text": "<image>"
         },
         {
@@ -9422,7 +9422,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.6796875,
+          "logprob": -11.6875,
           "text": "<image>"
         },
         {
@@ -9437,27 +9437,27 @@
         },
         {
           "id": 32000,
-          "logprob": -12.125,
+          "logprob": -12.1171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5703125,
+          "logprob": -12.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.109375,
+          "logprob": -12.1015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0078125,
+          "logprob": -12.0,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.734375,
+          "logprob": -13.7265625,
           "text": "<image>"
         },
         {
@@ -9467,67 +9467,67 @@
         },
         {
           "id": 32000,
-          "logprob": -11.6484375,
+          "logprob": -11.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.34375,
+          "logprob": -12.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.140625,
+          "logprob": -12.1484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1796875,
+          "logprob": -14.21875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.8046875,
+          "logprob": -14.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -18.046875,
+          "logprob": -17.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.34375,
+          "logprob": -15.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.4453125,
+          "logprob": -14.4765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.625,
+          "logprob": -10.6015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0546875,
+          "logprob": -12.0703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.859375,
+          "logprob": -12.84375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2265625,
+          "logprob": -12.2421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2109375,
+          "logprob": -13.203125,
           "text": "<image>"
         },
         {
@@ -9537,37 +9537,37 @@
         },
         {
           "id": 32000,
-          "logprob": -10.9296875,
+          "logprob": -10.9375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9140625,
+          "logprob": -11.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.125,
+          "logprob": -12.1015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9609375,
+          "logprob": -10.9453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7734375,
+          "logprob": -11.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.828125,
+          "logprob": -11.8203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2890625,
+          "logprob": -12.296875,
           "text": "<image>"
         },
         {
@@ -9577,27 +9577,27 @@
         },
         {
           "id": 32000,
-          "logprob": -11.53125,
+          "logprob": -11.5390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0234375,
+          "logprob": -12.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.2421875,
+          "logprob": -15.2265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1171875,
+          "logprob": -12.125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1171875,
+          "logprob": -12.125,
           "text": "<image>"
         },
         {
@@ -9607,7 +9607,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.671875,
+          "logprob": -11.6796875,
           "text": "<image>"
         },
         {
@@ -9627,32 +9627,32 @@
         },
         {
           "id": 32000,
-          "logprob": -12.734375,
+          "logprob": -12.703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.609375,
+          "logprob": -11.6328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5234375,
+          "logprob": -11.515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5078125,
+          "logprob": -11.546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2109375,
+          "logprob": -12.21875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.421875,
+          "logprob": -11.4140625,
           "text": "<image>"
         },
         {
@@ -9662,137 +9662,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.96875,
+          "logprob": -11.9765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.46875,
+          "logprob": -12.9609375,
           "text": "<image>"
         },
         {
@@ -9800,6 +9675,131 @@
           "logprob": -12.578125,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -11.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5078125,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -11.6171875,
@@ -9807,32 +9807,32 @@
         },
         {
           "id": 32000,
-          "logprob": -13.8125,
+          "logprob": -13.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1640625,
+          "logprob": -11.203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.765625,
+          "logprob": -11.78125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.5,
+          "logprob": -16.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3046875,
+          "logprob": -12.296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1171875,
+          "logprob": -14.1484375,
           "text": "<image>"
         },
         {
@@ -9842,12 +9842,47 @@
         },
         {
           "id": 32000,
-          "logprob": -12.5,
+          "logprob": -12.4921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.6015625,
+          "logprob": -10.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1953125,
           "text": "<image>"
         },
         {
@@ -9857,52 +9892,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.625,
+          "logprob": -11.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3046875,
+          "logprob": -12.6484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1171875,
+          "logprob": -11.109375,
           "text": "<image>"
         },
         {
@@ -9912,7 +9912,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.921875,
+          "logprob": -13.90625,
           "text": "<image>"
         },
         {
@@ -9927,12 +9927,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.3671875,
+          "logprob": -13.375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.7109375,
+          "logprob": -14.703125,
           "text": "<image>"
         },
         {
@@ -9942,7 +9942,7 @@
         },
         {
           "id": 32000,
-          "logprob": -14.859375,
+          "logprob": -14.890625,
           "text": "<image>"
         },
         {
@@ -9957,37 +9957,37 @@
         },
         {
           "id": 32000,
-          "logprob": -11.9296875,
+          "logprob": -11.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3828125,
+          "logprob": -13.3984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.046875,
+          "logprob": -13.9453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1484375,
+          "logprob": -13.1875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.265625,
+          "logprob": -11.28125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.125,
+          "logprob": -15.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.046875,
+          "logprob": -15.0234375,
           "text": "<image>"
         },
         {
@@ -10002,92 +10002,22 @@
         },
         {
           "id": 32000,
-          "logprob": -14.09375,
+          "logprob": -14.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.890625,
+          "logprob": -13.8984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.3203125,
+          "logprob": -15.3125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9609375,
+          "logprob": -12.8125,
           "text": "<image>"
         },
         {
@@ -10097,22 +10027,92 @@
         },
         {
           "id": 32000,
-          "logprob": -12.3203125,
+          "logprob": -13.046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.59375,
+          "logprob": -10.625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.171875,
+          "logprob": -11.25,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5,
+          "logprob": -12.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4921875,
           "text": "<image>"
         },
         {
@@ -10142,17 +10142,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.78125,
+          "logprob": -12.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3125,
+          "logprob": -11.3046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1171875,
+          "logprob": -11.109375,
           "text": "<image>"
         },
         {
@@ -10162,27 +10162,27 @@
         },
         {
           "id": 32000,
-          "logprob": -13.0859375,
+          "logprob": -13.0234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.921875,
+          "logprob": -10.9296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.515625,
+          "logprob": -13.5078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4921875,
+          "logprob": -11.5,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8203125,
+          "logprob": -11.8359375,
           "text": "<image>"
         },
         {
@@ -10192,42 +10192,42 @@
         },
         {
           "id": 32000,
-          "logprob": -14.0078125,
+          "logprob": -14.0234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.21875,
+          "logprob": -15.2578125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -17.84375,
+          "logprob": -17.953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.046875,
+          "logprob": -12.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.296875,
+          "logprob": -12.28125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7109375,
+          "logprob": -11.6875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1015625,
+          "logprob": -12.1171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.515625,
+          "logprob": -11.5546875,
           "text": "<image>"
         },
         {
@@ -10237,17 +10237,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.640625,
+          "logprob": -12.6171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4609375,
+          "logprob": -13.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.84375,
+          "logprob": -14.828125,
           "text": "<image>"
         },
         {
@@ -10257,22 +10257,22 @@
         },
         {
           "id": 32000,
-          "logprob": -11.6171875,
+          "logprob": -11.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.765625,
+          "logprob": -12.7421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5390625,
+          "logprob": -13.484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4453125,
+          "logprob": -11.4609375,
           "text": "<image>"
         },
         {
@@ -10282,37 +10282,37 @@
         },
         {
           "id": 32000,
-          "logprob": -11.96875,
+          "logprob": -11.984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4765625,
+          "logprob": -12.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.890625,
+          "logprob": -15.9375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.953125,
+          "logprob": -11.984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.40625,
+          "logprob": -11.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0,
+          "logprob": -12.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1875,
+          "logprob": -14.3046875,
           "text": "<image>"
         },
         {
@@ -10322,7 +10322,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.125,
+          "logprob": -12.1171875,
           "text": "<image>"
         },
         {
@@ -10332,22 +10332,22 @@
         },
         {
           "id": 32000,
-          "logprob": -13.140625,
+          "logprob": -13.1484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1484375,
+          "logprob": -12.1875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.375,
+          "logprob": -13.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.9453125,
+          "logprob": -13.9921875,
           "text": "<image>"
         },
         {
@@ -10357,12 +10357,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0859375,
+          "logprob": -12.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.734375,
+          "logprob": -10.7421875,
           "text": "<image>"
         },
         {
@@ -10372,12 +10372,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.65625,
+          "logprob": -12.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.953125,
+          "logprob": -11.9765625,
           "text": "<image>"
         },
         {
@@ -10387,12 +10387,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.5,
+          "logprob": -12.4765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8984375,
+          "logprob": -11.890625,
           "text": "<image>"
         },
         {
@@ -10412,22 +10412,22 @@
         },
         {
           "id": 32000,
-          "logprob": -12.4375,
+          "logprob": -12.4609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5703125,
+          "logprob": -12.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.828125,
+          "logprob": -12.8203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.375,
+          "logprob": -13.359375,
           "text": "<image>"
         },
         {
@@ -10447,7 +10447,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.2421875,
+          "logprob": -11.234375,
           "text": "<image>"
         },
         {
@@ -10457,27 +10457,27 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.328125,
+          "logprob": -15.3046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.890625,
+          "logprob": -12.8828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4453125,
+          "logprob": -13.453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8203125,
+          "logprob": -11.8125,
           "text": "<image>"
         },
         {
@@ -10487,17 +10487,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.96875,
+          "logprob": -12.9765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9921875,
+          "logprob": -11.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.421875,
+          "logprob": -13.4296875,
           "text": "<image>"
         },
         {
@@ -10507,12 +10507,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.265625,
+          "logprob": -13.28125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.265625,
+          "logprob": -13.2578125,
           "text": "<image>"
         },
         {
@@ -10527,12 +10527,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.15625,
+          "logprob": -12.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.375,
+          "logprob": -10.3828125,
           "text": "<image>"
         },
         {
@@ -10542,7 +10542,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.2109375,
+          "logprob": -13.203125,
           "text": "<image>"
         },
         {
@@ -10552,7 +10552,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.15625,
+          "logprob": -12.1328125,
           "text": "<image>"
         },
         {
@@ -10562,12 +10562,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.171875,
+          "logprob": -13.1796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6328125,
+          "logprob": -11.65625,
           "text": "<image>"
         },
         {
@@ -10577,12 +10577,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.90625,
+          "logprob": -11.9375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.765625,
+          "logprob": -11.7734375,
           "text": "<image>"
         },
         {
@@ -10597,7 +10597,7 @@
         },
         {
           "id": 32000,
-          "logprob": -10.8984375,
+          "logprob": -10.90625,
           "text": "<image>"
         },
         {
@@ -10607,17 +10607,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.40625,
+          "logprob": -12.3828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5390625,
+          "logprob": -11.5234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.59375,
+          "logprob": -16.578125,
           "text": "<image>"
         },
         {
@@ -10627,7 +10627,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.9140625,
+          "logprob": -11.9453125,
           "text": "<image>"
         },
         {
@@ -10642,22 +10642,22 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8984375,
+          "logprob": -11.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0703125,
+          "logprob": -13.0625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.03125,
+          "logprob": -12.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.921875,
+          "logprob": -11.9296875,
           "text": "<image>"
         },
         {
@@ -10667,12 +10667,12 @@
         },
         {
           "id": 32000,
-          "logprob": -10.5390625,
+          "logprob": -10.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.453125,
+          "logprob": -11.4375,
           "text": "<image>"
         },
         {
@@ -10682,22 +10682,22 @@
         },
         {
           "id": 32000,
-          "logprob": -13.0859375,
+          "logprob": -13.09375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1796875,
+          "logprob": -12.15625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.8203125,
+          "logprob": -15.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.984375,
+          "logprob": -10.9765625,
           "text": "<image>"
         },
         {
@@ -10707,42 +10707,42 @@
         },
         {
           "id": 32000,
-          "logprob": -10.109375,
+          "logprob": -10.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8671875,
+          "logprob": -11.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1328125,
+          "logprob": -13.125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.640625,
+          "logprob": -12.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.015625,
+          "logprob": -16.046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5078125,
+          "logprob": -11.515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.7265625,
+          "logprob": -13.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.703125,
+          "logprob": -14.671875,
           "text": "<image>"
         },
         {
@@ -10752,32 +10752,32 @@
         },
         {
           "id": 32000,
-          "logprob": -12.2421875,
+          "logprob": -12.2109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.6640625,
+          "logprob": -10.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.6640625,
+          "logprob": -15.6484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9140625,
+          "logprob": -10.9296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3203125,
+          "logprob": -11.328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.046875,
+          "logprob": -11.0546875,
           "text": "<image>"
         },
         {
@@ -10787,7 +10787,7 @@
         },
         {
           "id": 32000,
-          "logprob": -10.953125,
+          "logprob": -10.96875,
           "text": "<image>"
         },
         {
@@ -10802,337 +10802,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2578125,
+          "logprob": -12.359375,
           "text": "<image>"
         },
         {
@@ -11142,22 +10812,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.84375,
+          "logprob": -14.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
+          "logprob": -11.4921875,
           "text": "<image>"
         },
         {
@@ -11167,87 +10827,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.125,
+          "logprob": -12.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.390625,
+          "logprob": -12.640625,
           "text": "<image>"
         },
         {
@@ -11257,87 +10842,297 @@
         },
         {
           "id": 32000,
-          "logprob": -12.5078125,
+          "logprob": -12.4375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.09375,
+          "logprob": -11.8828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.8515625,
+          "logprob": -12.15625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8203125,
+          "logprob": -11.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.734375,
+          "logprob": -13.0,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2578125,
+          "logprob": -12.0703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.28125,
+          "logprob": -11.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.859375,
+          "logprob": -13.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.703125,
+          "logprob": -12.1484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5703125,
+          "logprob": -13.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.59375,
+          "logprob": -12.203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.890625,
+          "logprob": -11.9375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3359375,
+          "logprob": -12.1796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0625,
+          "logprob": -10.7421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.53125,
+          "logprob": -10.8203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7109375,
+          "logprob": -12.375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.7578125,
+          "logprob": -11.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.25,
           "text": "<image>"
         },
         {
@@ -11347,17 +11142,47 @@
         },
         {
           "id": 32000,
-          "logprob": -10.75,
+          "logprob": -11.8515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.2265625,
+          "logprob": -13.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9296875,
+          "logprob": -10.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1328125,
           "text": "<image>"
         },
         {
@@ -11367,7 +11192,12 @@
         },
         {
           "id": 32000,
-          "logprob": -10.5390625,
+          "logprob": -14.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0546875,
           "text": "<image>"
         },
         {
@@ -11375,6 +11205,56 @@
           "logprob": -11.6796875,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -11.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.234375,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -12.5390625,
@@ -11382,7 +11262,127 @@
         },
         {
           "id": 32000,
-          "logprob": -16.953125,
+          "logprob": -15.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.0,
           "text": "<image>"
         },
         {
@@ -11392,12 +11392,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.3359375,
+          "logprob": -11.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.671875,
+          "logprob": -11.6796875,
           "text": "<image>"
         },
         {
@@ -11412,12 +11412,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8671875,
+          "logprob": -11.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.484375,
+          "logprob": -11.4765625,
           "text": "<image>"
         },
         {
@@ -11427,12 +11427,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.6640625,
+          "logprob": -12.6875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4296875,
+          "logprob": -13.421875,
           "text": "<image>"
         },
         {
@@ -11447,7 +11447,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.3046875,
+          "logprob": -12.3125,
           "text": "<image>"
         },
         {
@@ -11457,17 +11457,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.34375,
+          "logprob": -13.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0703125,
+          "logprob": -13.0625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.234375,
+          "logprob": -11.2421875,
           "text": "<image>"
         },
         {
@@ -11477,37 +11477,37 @@
         },
         {
           "id": 32000,
-          "logprob": -13.4921875,
+          "logprob": -13.5,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.7578125,
+          "logprob": -10.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.53125,
+          "logprob": -13.5390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6953125,
+          "logprob": -12.703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.203125,
+          "logprob": -13.1953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -17.078125,
+          "logprob": -17.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.25,
+          "logprob": -15.0234375,
           "text": "<image>"
         },
         {
@@ -11517,17 +11517,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.140625,
+          "logprob": -12.125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5234375,
+          "logprob": -11.515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.0625,
+          "logprob": -16.125,
           "text": "<image>"
         },
         {
@@ -11537,17 +11537,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.6015625,
+          "logprob": -11.609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.5859375,
+          "logprob": -14.6015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.890625,
+          "logprob": -11.8984375,
           "text": "<image>"
         },
         {
@@ -11562,22 +11562,22 @@
         },
         {
           "id": 32000,
-          "logprob": -11.7734375,
+          "logprob": -11.78125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9921875,
+          "logprob": -13.0,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9375,
+          "logprob": -12.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9921875,
+          "logprob": -12.0,
           "text": "<image>"
         },
         {
@@ -11587,7 +11587,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.7578125,
+          "logprob": -11.7734375,
           "text": "<image>"
         },
         {
@@ -11597,7 +11597,7 @@
         },
         {
           "id": 32000,
-          "logprob": -14.328125,
+          "logprob": -14.3203125,
           "text": "<image>"
         },
         {
@@ -11607,22 +11607,22 @@
         },
         {
           "id": 32000,
-          "logprob": -13.84375,
+          "logprob": -13.875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.2109375,
+          "logprob": -14.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.390625,
+          "logprob": -12.375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.953125,
+          "logprob": -13.9375,
           "text": "<image>"
         },
         {
@@ -11632,27 +11632,27 @@
         },
         {
           "id": 32000,
-          "logprob": -14.6796875,
+          "logprob": -14.609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.84375,
+          "logprob": -11.8359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.65625,
+          "logprob": -10.6484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0078125,
+          "logprob": -12.0,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0859375,
+          "logprob": -11.0703125,
           "text": "<image>"
         },
         {
@@ -11672,7 +11672,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.6875,
+          "logprob": -12.6640625,
           "text": "<image>"
         },
         {
@@ -11687,22 +11687,22 @@
         },
         {
           "id": 32000,
-          "logprob": -16.109375,
+          "logprob": -16.09375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.984375,
+          "logprob": -11.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.40625,
+          "logprob": -13.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.375,
+          "logprob": -13.453125,
           "text": "<image>"
         },
         {
@@ -11712,57 +11712,57 @@
         },
         {
           "id": 32000,
-          "logprob": -12.921875,
+          "logprob": -12.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.859375,
+          "logprob": -14.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3671875,
+          "logprob": -13.375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0546875,
+          "logprob": -12.0703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.21875,
+          "logprob": -15.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6953125,
+          "logprob": -11.6875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.5234375,
+          "logprob": -10.5390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.8984375,
+          "logprob": -14.703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.8515625,
+          "logprob": -13.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6796875,
+          "logprob": -11.6875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -17.4375,
+          "logprob": -17.453125,
           "text": "<image>"
         },
         {
@@ -11777,7 +11777,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8125,
+          "logprob": -11.7890625,
           "text": "<image>"
         },
         {
@@ -11792,12 +11792,12 @@
         },
         {
           "id": 32000,
-          "logprob": -14.2421875,
+          "logprob": -14.265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.7265625,
+          "logprob": -13.6953125,
           "text": "<image>"
         },
         {
@@ -11812,57 +11812,57 @@
         },
         {
           "id": 32000,
-          "logprob": -12.84375,
+          "logprob": -12.890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9140625,
+          "logprob": -11.9375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.7421875,
+          "logprob": -14.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.984375,
+          "logprob": -11.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.71875,
+          "logprob": -14.7265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.578125,
+          "logprob": -11.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.015625,
+          "logprob": -13.0,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.796875,
+          "logprob": -12.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0078125,
+          "logprob": -11.015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.46875,
+          "logprob": -16.421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9296875,
+          "logprob": -11.921875,
           "text": "<image>"
         },
         {
@@ -11872,7 +11872,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.8515625,
+          "logprob": -12.859375,
           "text": "<image>"
         },
         {
@@ -11887,12 +11887,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.140625,
+          "logprob": -12.1328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.359375,
+          "logprob": -13.3515625,
           "text": "<image>"
         },
         {
@@ -11907,27 +11907,27 @@
         },
         {
           "id": 32000,
-          "logprob": -11.578125,
+          "logprob": -11.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8203125,
+          "logprob": -12.84375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.984375,
+          "logprob": -11.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5078125,
+          "logprob": -12.515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.625,
+          "logprob": -12.6328125,
           "text": "<image>"
         },
         {
@@ -11942,7 +11942,7 @@
         },
         {
           "id": 32000,
-          "logprob": -14.0546875,
+          "logprob": -14.046875,
           "text": "<image>"
         },
         {
@@ -11952,7 +11952,7 @@
         },
         {
           "id": 32000,
-          "logprob": -17.21875,
+          "logprob": -17.203125,
           "text": "<image>"
         },
         {
@@ -11962,17 +11962,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.3671875,
+          "logprob": -13.375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.6171875,
+          "logprob": -14.609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9609375,
+          "logprob": -12.953125,
           "text": "<image>"
         },
         {
@@ -11982,12 +11982,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.7265625,
+          "logprob": -11.71875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.03125,
+          "logprob": -15.0234375,
           "text": "<image>"
         },
         {
@@ -11997,7 +11997,7 @@
         },
         {
           "id": 32000,
-          "logprob": -15.890625,
+          "logprob": -15.9296875,
           "text": "<image>"
         },
         {
@@ -12022,22 +12022,22 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0234375,
+          "logprob": -12.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7578125,
+          "logprob": -12.75,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.7578125,
+          "logprob": -14.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5,
+          "logprob": -11.484375,
           "text": "<image>"
         },
         {
@@ -12047,7 +12047,7 @@
         },
         {
           "id": 32000,
-          "logprob": -10.7265625,
+          "logprob": -10.734375,
           "text": "<image>"
         },
         {
@@ -12057,22 +12057,22 @@
         },
         {
           "id": 32000,
-          "logprob": -11.4140625,
+          "logprob": -11.421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9453125,
+          "logprob": -12.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1171875,
+          "logprob": -13.0625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0390625,
+          "logprob": -12.03125,
           "text": "<image>"
         },
         {
@@ -12082,7 +12082,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.234375,
+          "logprob": -12.2109375,
           "text": "<image>"
         },
         {
@@ -12097,7 +12097,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.0390625,
+          "logprob": -13.09375,
           "text": "<image>"
         },
         {
@@ -12107,7 +12107,7 @@
         },
         {
           "id": 32000,
-          "logprob": -15.265625,
+          "logprob": -15.2578125,
           "text": "<image>"
         },
         {
@@ -12120,2201 +12120,6 @@
           "logprob": -11.0546875,
           "text": "<image>"
         },
-        {
-          "id": 32000,
-          "logprob": -11.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -19.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -18.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -18.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.7734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -19.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0625,
-          "text": "<image>"
-        },
         {
           "id": 32000,
           "logprob": -11.4609375,
@@ -14322,841 +12127,7 @@
         },
         {
           "id": 32000,
-          "logprob": -14.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.1953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5,
-          "text": "<image>"
-        },
-        {
-          "id": 2418,
-          "logprob": -19.0625,
-          "text": "Can"
-        },
-        {
-          "id": 368,
-          "logprob": -0.19726562,
-          "text": "you"
-        },
-        {
-          "id": 1912,
-          "logprob": -1.4990234,
-          "text": "tell"
-        },
-        {
-          "id": 528,
-          "logprob": -0.31152344,
-          "text": "me"
-        },
-        {
-          "id": 264,
-          "logprob": -2.6367188,
-          "text": "a"
-        },
-        {
-          "id": 1215,
-          "logprob": -9.1015625,
-          "text": "very"
-        },
-        {
-          "id": 2485,
-          "logprob": -0.9941406,
-          "text": "short"
-        },
-        {
-          "id": 2838,
-          "logprob": -0.46118164,
-          "text": "story"
-        },
-        {
-          "id": 2818,
-          "logprob": -3.3183594,
-          "text": "based"
-        },
-        {
-          "id": 356,
-          "logprob": -0.029129028,
-          "text": "on"
-        },
-        {
-          "id": 272,
-          "logprob": -0.9902344,
-          "text": "the"
-        },
-        {
-          "id": 3469,
-          "logprob": -0.29052734,
-          "text": "image"
-        },
-        {
-          "id": 28804,
-          "logprob": -0.43188477,
-          "text": "?"
-        }
-      ],
-      "seed": null,
-      "tokens": [
-        {
-          "id": 13,
-          "logprob": -0.0076828003,
-          "special": false,
-          "text": "\n"
-        },
-        {
-          "id": 13,
-          "logprob": -0.20092773,
-          "special": false,
-          "text": "\n"
-        },
-        {
-          "id": 16114,
-          "logprob": -1.2587891,
-          "special": false,
-          "text": "Once"
-        },
-        {
-          "id": 3714,
-          "logprob": -0.20861816,
-          "special": false,
-          "text": " upon"
-        },
-        {
-          "id": 264,
-          "logprob": -0.0017719269,
-          "special": false,
-          "text": " a"
-        },
-        {
-          "id": 727,
-          "logprob": -0.011909485,
-          "special": false,
-          "text": " time"
-        },
-        {
-          "id": 28725,
-          "logprob": -0.17529297,
-          "special": false,
-          "text": ","
-        },
-        {
-          "id": 736,
-          "logprob": -0.9082031,
-          "special": false,
-          "text": " there"
-        },
-        {
-          "id": 403,
-          "logprob": -0.057525635,
-          "special": false,
-          "text": " was"
-        },
-        {
-          "id": 264,
-          "logprob": -0.009651184,
-          "special": false,
-          "text": " a"
-        }
-      ],
-      "top_tokens": null
-    },
-    "generated_text": "\n\nOnce upon a time, there was a"
-  },
-  {
-    "details": {
-      "best_of_sequences": null,
-      "finish_reason": "length",
-      "generated_tokens": 10,
-      "prefill": [
-        {
-          "id": 1,
-          "logprob": null,
-          "text": "<s>"
-        },
-        {
-          "id": 1247,
-          "logprob": -2.3886719,
-          "text": "User"
-        },
-        {
-          "id": 28747,
-          "logprob": -12.328125,
-          "text": ":"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.7109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -18.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -9.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.7734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.7421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -18.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.7109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.2578125,
+          "logprob": -11.8046875,
           "text": "<image>"
         },
         {
@@ -15164,2156 +12135,11 @@
           "logprob": -11.140625,
           "text": "<image>"
         },
-        {
-          "id": 32000,
-          "logprob": -13.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.7890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.3671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.7109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.640625,
-          "text": "<image>"
-        },
         {
           "id": 32000,
           "logprob": -10.6875,
           "text": "<image>"
         },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -9.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.4609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.7421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.7890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.1875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5859375,
-          "text": "<image>"
-        },
         {
           "id": 32000,
           "logprob": -12.0546875,
@@ -17321,557 +12147,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.7890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -9.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7265625,
+          "logprob": -17.984375,
           "text": "<image>"
         },
         {
@@ -17879,1251 +12155,6 @@
           "logprob": -12.4140625,
           "text": "<image>"
         },
-        {
-          "id": 32000,
-          "logprob": -11.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -9.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.7890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -18.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.46875,
-          "text": "<image>"
-        },
         {
           "id": 32000,
           "logprob": -14.703125,
@@ -19131,122 +12162,7 @@
         },
         {
           "id": 32000,
-          "logprob": -14.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
+          "logprob": -11.796875,
           "text": "<image>"
         },
         {
@@ -19256,207 +12172,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.328125,
+          "logprob": -11.7109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6484375,
+          "logprob": -12.828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7578125,
+          "logprob": -12.8359375,
           "text": "<image>"
         },
         {
@@ -19466,237 +12192,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.4765625,
+          "logprob": -13.5390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6328125,
+          "logprob": -16.21875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.625,
+          "logprob": -16.84375,
           "text": "<image>"
         },
         {
@@ -19706,947 +12212,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.3515625,
+          "logprob": -14.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.7890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0234375,
+          "logprob": -11.71875,
           "text": "<image>"
         },
         {
@@ -20656,887 +12227,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.4609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.7890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7890625,
+          "logprob": -12.984375,
           "text": "<image>"
         },
         {
@@ -21546,32 +12237,112 @@
         },
         {
           "id": 32000,
-          "logprob": -13.4140625,
+          "logprob": -12.21875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.96875,
+          "logprob": -17.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1171875,
+          "logprob": -12.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5390625,
+          "logprob": -15.046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.609375,
+          "logprob": -16.71875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6484375,
+          "logprob": -11.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5625,
           "text": "<image>"
         },
         {
@@ -21581,32 +12352,1092 @@
         },
         {
           "id": 32000,
-          "logprob": -12.9921875,
+          "logprob": -11.9453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4453125,
+          "logprob": -11.8359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.9140625,
+          "logprob": -11.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.7109375,
+          "logprob": -12.875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.625,
+          "logprob": -11.9609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.765625,
+          "logprob": -13.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -19.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -18.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.0859375,
           "text": "<image>"
         },
         {
@@ -21619,6641 +13450,16 @@
           "logprob": -14.53125,
           "text": "<image>"
         },
-        {
-          "id": 32000,
-          "logprob": -13.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9296875,
-          "text": "<image>"
-        },
         {
           "id": 32000,
           "logprob": -12.359375,
           "text": "<image>"
         },
-        {
-          "id": 32000,
-          "logprob": -11.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.7109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1328125,
-          "text": "<image>"
-        },
         {
           "id": 32000,
           "logprob": -12.0625,
           "text": "<image>"
         },
-        {
-          "id": 32000,
-          "logprob": -11.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.1875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.7109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -9.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.7734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -18.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.7109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -9.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.7421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -19.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -18.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0390625,
-          "text": "<image>"
-        },
         {
           "id": 32000,
           "logprob": -13.640625,
@@ -28261,42 +13467,42 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8359375,
+          "logprob": -11.8203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1015625,
+          "logprob": -13.046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3515625,
+          "logprob": -13.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.578125,
+          "logprob": -12.625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.25,
+          "logprob": -11.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.15625,
+          "logprob": -14.2109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7265625,
+          "logprob": -11.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.90625,
+          "logprob": -13.8515625,
           "text": "<image>"
         },
         {
@@ -28306,12 +13512,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.6953125,
+          "logprob": -11.703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.5625,
+          "logprob": -10.65625,
           "text": "<image>"
         },
         {
@@ -28321,12 +13527,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.2265625,
+          "logprob": -12.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9296875,
+          "logprob": -11.8984375,
           "text": "<image>"
         },
         {
@@ -28336,12 +13542,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.875,
+          "logprob": -12.890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4453125,
+          "logprob": -11.453125,
           "text": "<image>"
         },
         {
@@ -28349,6 +13555,21 @@
           "logprob": -14.1796875,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -11.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.125,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -11.9453125,
@@ -28356,22 +13577,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5078125,
+          "logprob": -12.53125,
           "text": "<image>"
         },
         {
@@ -28381,27 +13587,27 @@
         },
         {
           "id": 32000,
-          "logprob": -12.2421875,
+          "logprob": -12.2265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3203125,
+          "logprob": -12.3046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.546875,
+          "logprob": -13.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.671875,
+          "logprob": -16.6875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.953125,
+          "logprob": -11.96875,
           "text": "<image>"
         },
         {
@@ -28411,7 +13617,7 @@
         },
         {
           "id": 32000,
-          "logprob": -14.9453125,
+          "logprob": -14.9140625,
           "text": "<image>"
         },
         {
@@ -28421,37 +13627,67 @@
         },
         {
           "id": 32000,
-          "logprob": -15.203125,
+          "logprob": -15.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.6953125,
+          "logprob": -14.6796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0234375,
+          "logprob": -12.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9453125,
+          "logprob": -12.9375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1796875,
+          "logprob": -12.2109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.75,
+          "logprob": -12.7421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3671875,
+          "logprob": -13.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6171875,
           "text": "<image>"
         },
         {
@@ -28461,42 +13697,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.4765625,
+          "logprob": -12.7109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3125,
+          "logprob": -13.2578125,
           "text": "<image>"
         },
         {
@@ -28506,12 +13712,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.484375,
+          "logprob": -13.5390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2734375,
+          "logprob": -13.265625,
           "text": "<image>"
         },
         {
@@ -28526,17 +13732,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.6796875,
+          "logprob": -11.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.53125,
+          "logprob": -16.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1484375,
+          "logprob": -14.171875,
           "text": "<image>"
         },
         {
@@ -28546,17 +13752,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8125,
+          "logprob": -11.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.03125,
+          "logprob": -15.1171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.484375,
+          "logprob": -15.4453125,
           "text": "<image>"
         },
         {
@@ -28571,37 +13777,37 @@
         },
         {
           "id": 32000,
-          "logprob": -10.6953125,
+          "logprob": -10.703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1640625,
+          "logprob": -11.15625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.84375,
+          "logprob": -12.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5625,
+          "logprob": -12.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7421875,
+          "logprob": -12.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -18.0,
+          "logprob": -17.953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5859375,
+          "logprob": -11.59375,
           "text": "<image>"
         },
         {
@@ -28616,12 +13822,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.6796875,
+          "logprob": -11.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.71875,
+          "logprob": -12.8515625,
           "text": "<image>"
         },
         {
@@ -28631,27 +13837,27 @@
         },
         {
           "id": 32000,
-          "logprob": -12.1875,
+          "logprob": -12.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7578125,
+          "logprob": -12.7265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.125,
+          "logprob": -15.140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.9140625,
+          "logprob": -14.9375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.546875,
+          "logprob": -15.6875,
           "text": "<image>"
         },
         {
@@ -28661,22 +13867,22 @@
         },
         {
           "id": 32000,
-          "logprob": -14.109375,
+          "logprob": -14.125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.234375,
+          "logprob": -13.2265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.15625,
+          "logprob": -13.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.734375,
+          "logprob": -12.703125,
           "text": "<image>"
         },
         {
@@ -28686,7 +13892,7 @@
         },
         {
           "id": 32000,
-          "logprob": -14.7734375,
+          "logprob": -14.765625,
           "text": "<image>"
         },
         {
@@ -28696,12 +13902,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.7421875,
+          "logprob": -11.71875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.921875,
+          "logprob": -11.9296875,
           "text": "<image>"
         },
         {
@@ -28711,17 +13917,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.6015625,
+          "logprob": -13.6328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4375,
+          "logprob": -11.4296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5,
+          "logprob": -13.5078125,
           "text": "<image>"
         },
         {
@@ -28736,27 +13942,27 @@
         },
         {
           "id": 32000,
-          "logprob": -12.1328125,
+          "logprob": -12.15625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.90625,
+          "logprob": -13.1171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.578125,
+          "logprob": -12.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3359375,
+          "logprob": -13.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7265625,
+          "logprob": -11.734375,
           "text": "<image>"
         },
         {
@@ -28764,491 +13970,11 @@
           "logprob": -10.953125,
           "text": "<image>"
         },
-        {
-          "id": 32000,
-          "logprob": -12.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -19.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.796875,
-          "text": "<image>"
-        },
         {
           "id": 32000,
           "logprob": -12.75,
           "text": "<image>"
         },
-        {
-          "id": 32000,
-          "logprob": -14.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.1953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.515625,
-          "text": "<image>"
-        },
         {
           "id": 32000,
           "logprob": -12.7734375,
@@ -29256,7 +13982,22 @@
         },
         {
           "id": 32000,
-          "logprob": -12.4609375,
+          "logprob": -13.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5390625,
           "text": "<image>"
         },
         {
@@ -29266,87 +14007,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.171875,
+          "logprob": -19.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8828125,
+          "logprob": -12.4765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8828125,
+          "logprob": -11.609375,
           "text": "<image>"
         },
         {
@@ -29359,6 +14030,541 @@
           "logprob": -13.3203125,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -12.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.359375,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -13.2109375,
@@ -29366,7 +14572,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.9765625,
+          "logprob": -11.984375,
           "text": "<image>"
         },
         {
@@ -29376,12 +14582,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0078125,
+          "logprob": -12.015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0390625,
+          "logprob": -11.03125,
           "text": "<image>"
         },
         {
@@ -29396,17 +14602,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.8203125,
+          "logprob": -13.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5078125,
+          "logprob": -13.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.734375,
+          "logprob": -11.7265625,
           "text": "<image>"
         },
         {
@@ -29416,22 +14622,22 @@
         },
         {
           "id": 32000,
-          "logprob": -17.3125,
+          "logprob": -16.84375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5234375,
+          "logprob": -12.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -17.625,
+          "logprob": -17.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9296875,
+          "logprob": -11.9453125,
           "text": "<image>"
         },
         {
@@ -29441,17 +14647,17 @@
         },
         {
           "id": 32000,
-          "logprob": -15.9140625,
+          "logprob": -16.0,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.65625,
+          "logprob": -16.625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5,
+          "logprob": -12.4921875,
           "text": "<image>"
         },
         {
@@ -29461,17 +14667,17 @@
         },
         {
           "id": 368,
-          "logprob": -0.19726562,
+          "logprob": -0.19604492,
           "text": "you"
         },
         {
           "id": 1912,
-          "logprob": -1.4990234,
+          "logprob": -1.5058594,
           "text": "tell"
         },
         {
           "id": 528,
-          "logprob": -0.31152344,
+          "logprob": -0.31030273,
           "text": "me"
         },
         {
@@ -29486,22 +14692,22 @@
         },
         {
           "id": 2485,
-          "logprob": -0.9941406,
+          "logprob": -0.9975586,
           "text": "short"
         },
         {
           "id": 2838,
-          "logprob": -0.46118164,
+          "logprob": -0.4633789,
           "text": "story"
         },
         {
           "id": 2818,
-          "logprob": -3.3183594,
+          "logprob": -3.3144531,
           "text": "based"
         },
         {
           "id": 356,
-          "logprob": -0.029129028,
+          "logprob": -0.029037476,
           "text": "on"
         },
         {
@@ -29511,12 +14717,12 @@
         },
         {
           "id": 3469,
-          "logprob": -0.29052734,
+          "logprob": -0.2890625,
           "text": "image"
         },
         {
           "id": 28804,
-          "logprob": -0.43188477,
+          "logprob": -0.42895508,
           "text": "?"
         }
       ],
@@ -29524,13 +14730,13 @@
       "tokens": [
         {
           "id": 13,
-          "logprob": -0.0076828003,
+          "logprob": -0.007621765,
           "special": false,
           "text": "\n"
         },
         {
           "id": 13,
-          "logprob": -0.19958496,
+          "logprob": -0.20812988,
           "special": false,
           "text": "\n"
         },
@@ -29542,37 +14748,37 @@
         },
         {
           "id": 3714,
-          "logprob": -0.20861816,
+          "logprob": -0.20825195,
           "special": false,
           "text": " upon"
         },
         {
           "id": 264,
-          "logprob": -0.0017719269,
+          "logprob": -0.0017709732,
           "special": false,
           "text": " a"
         },
         {
           "id": 727,
-          "logprob": -0.011749268,
+          "logprob": -0.011932373,
           "special": false,
           "text": " time"
         },
         {
           "id": 28725,
-          "logprob": -0.17529297,
+          "logprob": -0.17297363,
           "special": false,
           "text": ","
         },
         {
           "id": 736,
-          "logprob": -0.9086914,
+          "logprob": -0.9057617,
           "special": false,
           "text": " there"
         },
         {
           "id": 403,
-          "logprob": -0.056732178,
+          "logprob": -0.05758667,
           "special": false,
           "text": " was"
         },
@@ -29600,7 +14806,7 @@
         },
         {
           "id": 1247,
-          "logprob": -2.3886719,
+          "logprob": -2.390625,
           "text": "User"
         },
         {
@@ -29615,12 +14821,12 @@
         },
         {
           "id": 32000,
-          "logprob": -10.671875,
+          "logprob": -10.6953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.7109375,
+          "logprob": -15.828125,
           "text": "<image>"
         },
         {
@@ -29630,82 +14836,82 @@
         },
         {
           "id": 32000,
-          "logprob": -10.0234375,
+          "logprob": -10.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.1328125,
+          "logprob": -10.140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.421875,
+          "logprob": -10.3125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.90625,
+          "logprob": -12.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.59375,
+          "logprob": -15.4921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.828125,
+          "logprob": -13.8671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.390625,
+          "logprob": -11.3671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.1171875,
+          "logprob": -10.1015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.1640625,
+          "logprob": -10.1796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.234375,
+          "logprob": -10.2421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.3984375,
+          "logprob": -10.4609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.015625,
+          "logprob": -14.34375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0859375,
+          "logprob": -13.1796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2734375,
+          "logprob": -13.328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.359375,
+          "logprob": -14.3515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0390625,
+          "logprob": -11.0546875,
           "text": "<image>"
         },
         {
@@ -29715,12 +14921,12 @@
         },
         {
           "id": 32000,
-          "logprob": -10.5234375,
+          "logprob": -10.515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.4765625,
+          "logprob": -10.4453125,
           "text": "<image>"
         },
         {
@@ -29730,22 +14936,22 @@
         },
         {
           "id": 32000,
-          "logprob": -13.6171875,
+          "logprob": -13.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.359375,
+          "logprob": -11.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.8359375,
+          "logprob": -10.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -17.34375,
+          "logprob": -17.234375,
           "text": "<image>"
         },
         {
@@ -29760,12 +14966,12 @@
         },
         {
           "id": 32000,
-          "logprob": -10.640625,
+          "logprob": -10.6484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -18.390625,
+          "logprob": -17.984375,
           "text": "<image>"
         },
         {
@@ -29775,17 +14981,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.5625,
+          "logprob": -11.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -9.875,
+          "logprob": -9.8671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.7734375,
+          "logprob": -10.7578125,
           "text": "<image>"
         },
         {
@@ -29795,32 +15001,32 @@
         },
         {
           "id": 32000,
-          "logprob": -10.96875,
+          "logprob": -10.984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.609375,
+          "logprob": -10.625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.09375,
+          "logprob": -11.109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.5078125,
+          "logprob": -10.5,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4453125,
+          "logprob": -11.4609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.59375,
+          "logprob": -13.09375,
           "text": "<image>"
         },
         {
@@ -29830,132 +15036,87 @@
         },
         {
           "id": 32000,
-          "logprob": -10.5625,
+          "logprob": -10.5390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.640625,
+          "logprob": -10.625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9765625,
+          "logprob": -10.9296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.765625,
+          "logprob": -11.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3671875,
+          "logprob": -12.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0234375,
+          "logprob": -12.125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.59375,
+          "logprob": -10.578125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.7421875,
+          "logprob": -10.7265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0625,
+          "logprob": -11.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.3828125,
+          "logprob": -10.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.171875,
+          "logprob": -11.1796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0234375,
+          "logprob": -11.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -18.40625,
+          "logprob": -18.703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9921875,
+          "logprob": -11.015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.7109375,
+          "logprob": -15.6796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.15625,
+          "logprob": -12.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.140625,
+          "logprob": -10.4140625,
           "text": "<image>"
         },
         {
@@ -29965,12 +15126,42 @@
         },
         {
           "id": 32000,
-          "logprob": -10.7890625,
+          "logprob": -11.8046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4296875,
+          "logprob": -10.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.140625,
           "text": "<image>"
         },
         {
@@ -29980,12 +15171,27 @@
         },
         {
           "id": 32000,
-          "logprob": -10.796875,
+          "logprob": -10.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1640625,
+          "logprob": -11.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.15625,
           "text": "<image>"
         },
         {
@@ -29995,27 +15201,27 @@
         },
         {
           "id": 32000,
-          "logprob": -15.4453125,
+          "logprob": -15.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.2109375,
+          "logprob": -10.2265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.09375,
+          "logprob": -11.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.6796875,
+          "logprob": -14.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.3671875,
+          "logprob": -10.375,
           "text": "<image>"
         },
         {
@@ -30030,22 +15236,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.484375,
+          "logprob": -12.4921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.09375,
+          "logprob": -12.1015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.96875,
+          "logprob": -11.0859375,
           "text": "<image>"
         },
         {
@@ -30053,6 +15254,11 @@
           "logprob": -10.9765625,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -10.921875,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -11.0234375,
@@ -30068,11 +15274,6 @@
           "logprob": -11.1484375,
           "text": "<image>"
         },
-        {
-          "id": 32000,
-          "logprob": -10.703125,
-          "text": "<image>"
-        },
         {
           "id": 32000,
           "logprob": -10.71875,
@@ -30080,17 +15281,22 @@
         },
         {
           "id": 32000,
-          "logprob": -11.6875,
+          "logprob": -10.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.8984375,
+          "logprob": -11.6796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2890625,
+          "logprob": -10.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.375,
           "text": "<image>"
         },
         {
@@ -30100,12 +15306,12 @@
         },
         {
           "id": 32000,
-          "logprob": -10.640625,
+          "logprob": -10.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.7109375,
+          "logprob": -10.7265625,
           "text": "<image>"
         },
         {
@@ -30120,17 +15326,17 @@
         },
         {
           "id": 32000,
-          "logprob": -10.6875,
+          "logprob": -10.6796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.84375,
+          "logprob": -11.828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5078125,
+          "logprob": -11.5390625,
           "text": "<image>"
         },
         {
@@ -30140,22 +15346,22 @@
         },
         {
           "id": 32000,
-          "logprob": -10.9609375,
+          "logprob": -10.9765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4921875,
+          "logprob": -12.5,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5546875,
+          "logprob": -11.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.2265625,
+          "logprob": -11.2421875,
           "text": "<image>"
         },
         {
@@ -30165,7 +15371,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.21875,
+          "logprob": -11.2265625,
           "text": "<image>"
         },
         {
@@ -30175,52 +15381,52 @@
         },
         {
           "id": 32000,
-          "logprob": -11.0,
+          "logprob": -10.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0234375,
+          "logprob": -12.046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.09375,
+          "logprob": -11.109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.046875,
+          "logprob": -14.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.921875,
+          "logprob": -10.8984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9609375,
+          "logprob": -10.953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9140625,
+          "logprob": -10.9296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8125,
+          "logprob": -11.78125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.140625,
+          "logprob": -15.1484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9609375,
+          "logprob": -10.96875,
           "text": "<image>"
         },
         {
@@ -30230,72 +15436,72 @@
         },
         {
           "id": 32000,
-          "logprob": -10.8828125,
+          "logprob": -10.890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9609375,
+          "logprob": -10.9453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.46875,
+          "logprob": -11.453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5234375,
+          "logprob": -13.09375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.328125,
+          "logprob": -12.484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.375,
+          "logprob": -11.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3515625,
+          "logprob": -12.3359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.578125,
+          "logprob": -11.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5078125,
+          "logprob": -12.5,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.640625,
+          "logprob": -12.6328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1953125,
+          "logprob": -11.203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9921875,
+          "logprob": -11.9765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.921875,
+          "logprob": -10.9296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.7578125,
+          "logprob": -10.7421875,
           "text": "<image>"
         },
         {
@@ -30305,12 +15511,12 @@
         },
         {
           "id": 32000,
-          "logprob": -15.015625,
+          "logprob": -15.0234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7734375,
+          "logprob": -12.75,
           "text": "<image>"
         },
         {
@@ -30325,27 +15531,27 @@
         },
         {
           "id": 32000,
-          "logprob": -11.2890625,
+          "logprob": -11.265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.34375,
+          "logprob": -13.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.6953125,
+          "logprob": -10.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.59375,
+          "logprob": -10.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6015625,
+          "logprob": -11.7578125,
           "text": "<image>"
         },
         {
@@ -30360,7 +15566,7 @@
         },
         {
           "id": 32000,
-          "logprob": -10.5390625,
+          "logprob": -10.5859375,
           "text": "<image>"
         },
         {
@@ -30370,12 +15576,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.9765625,
+          "logprob": -17.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4609375,
+          "logprob": -11.4765625,
           "text": "<image>"
         },
         {
@@ -30385,12 +15591,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.015625,
+          "logprob": -12.0078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.84375,
+          "logprob": -12.8359375,
           "text": "<image>"
         },
         {
@@ -30400,12 +15606,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.4375,
+          "logprob": -12.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.671875,
+          "logprob": -12.6875,
           "text": "<image>"
         },
         {
@@ -30415,57 +15621,27 @@
         },
         {
           "id": 32000,
-          "logprob": -12.875,
+          "logprob": -12.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2578125,
+          "logprob": -12.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.359375,
+          "logprob": -10.3359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.375,
+          "logprob": -13.46875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.7421875,
+          "logprob": -11.8671875,
           "text": "<image>"
         },
         {
@@ -30475,7 +15651,32 @@
         },
         {
           "id": 32000,
-          "logprob": -11.0390625,
+          "logprob": -11.0078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8203125,
           "text": "<image>"
         },
         {
@@ -30485,17 +15686,22 @@
         },
         {
           "id": 32000,
-          "logprob": -10.6171875,
+          "logprob": -10.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2421875,
+          "logprob": -10.6328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.8359375,
+          "logprob": -12.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8203125,
           "text": "<image>"
         },
         {
@@ -30515,127 +15721,62 @@
         },
         {
           "id": 32000,
-          "logprob": -11.0,
+          "logprob": -10.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7734375,
+          "logprob": -12.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2109375,
+          "logprob": -12.21875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4296875,
+          "logprob": -11.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2421875,
+          "logprob": -12.828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3046875,
+          "logprob": -11.328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.2890625,
+          "logprob": -10.296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.8203125,
+          "logprob": -10.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9140625,
+          "logprob": -11.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.2421875,
+          "logprob": -11.21875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.234375,
+          "logprob": -11.2109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4375,
+          "logprob": -11.5078125,
           "text": "<image>"
         },
         {
@@ -30645,27 +15786,12 @@
         },
         {
           "id": 32000,
-          "logprob": -14.203125,
+          "logprob": -10.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.59375,
+          "logprob": -10.3515625,
           "text": "<image>"
         },
         {
@@ -30675,102 +15801,22 @@
         },
         {
           "id": 32000,
-          "logprob": -10.90625,
+          "logprob": -10.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.2578125,
+          "logprob": -10.828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.921875,
+          "logprob": -11.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.890625,
+          "logprob": -11.9375,
           "text": "<image>"
         },
         {
@@ -30780,112 +15826,32 @@
         },
         {
           "id": 32000,
-          "logprob": -10.9609375,
+          "logprob": -12.3046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.7890625,
+          "logprob": -11.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4453125,
+          "logprob": -12.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.390625,
+          "logprob": -13.3984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1640625,
+          "logprob": -11.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0703125,
+          "logprob": -14.1875,
           "text": "<image>"
         },
         {
@@ -30895,72 +15861,162 @@
         },
         {
           "id": 32000,
-          "logprob": -12.859375,
+          "logprob": -12.421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1875,
+          "logprob": -10.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.171875,
+          "logprob": -10.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6640625,
+          "logprob": -10.6015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1953125,
+          "logprob": -10.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1328125,
+          "logprob": -11.2265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1875,
+          "logprob": -10.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9453125,
+          "logprob": -10.9765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8515625,
+          "logprob": -12.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.203125,
+          "logprob": -10.8671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1796875,
+          "logprob": -10.8515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9609375,
+          "logprob": -11.8984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.703125,
+          "logprob": -10.7421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8515625,
+          "logprob": -13.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1796875,
           "text": "<image>"
         },
         {
@@ -30968,11 +16024,161 @@
           "logprob": -11.75,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -10.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.640625,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -11.8359375,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -11.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.828125,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -12.1953125,
@@ -30980,32 +16186,32 @@
         },
         {
           "id": 32000,
-          "logprob": -12.5078125,
+          "logprob": -12.4921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.546875,
+          "logprob": -11.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.078125,
+          "logprob": -12.0859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.2421875,
+          "logprob": -11.265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6640625,
+          "logprob": -11.6953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2109375,
+          "logprob": -12.21875,
           "text": "<image>"
         },
         {
@@ -31025,27 +16231,7 @@
         },
         {
           "id": 32000,
-          "logprob": -15.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3046875,
+          "logprob": -15.03125,
           "text": "<image>"
         },
         {
@@ -31055,32 +16241,52 @@
         },
         {
           "id": 32000,
-          "logprob": -12.2265625,
+          "logprob": -11.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9453125,
+          "logprob": -13.1171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4921875,
+          "logprob": -12.296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.8828125,
+          "logprob": -12.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3125,
+          "logprob": -12.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8984375,
+          "logprob": -12.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9140625,
           "text": "<image>"
         },
         {
@@ -31090,12 +16296,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8125,
+          "logprob": -11.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.90625,
+          "logprob": -11.9140625,
           "text": "<image>"
         },
         {
@@ -31115,12 +16321,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.140625,
+          "logprob": -13.1484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9765625,
+          "logprob": -11.9921875,
           "text": "<image>"
         },
         {
@@ -31135,7 +16341,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.84375,
+          "logprob": -11.8359375,
           "text": "<image>"
         },
         {
@@ -31150,17 +16356,22 @@
         },
         {
           "id": 32000,
-          "logprob": -13.1953125,
+          "logprob": -13.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.875,
+          "logprob": -11.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.6015625,
+          "logprob": -13.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6484375,
           "text": "<image>"
         },
         {
@@ -31170,27 +16381,22 @@
         },
         {
           "id": 32000,
-          "logprob": -13.671875,
+          "logprob": -11.4609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5390625,
+          "logprob": -13.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.59375,
+          "logprob": -12.4375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5703125,
+          "logprob": -14.6328125,
           "text": "<image>"
         },
         {
@@ -31200,57 +16406,57 @@
         },
         {
           "id": 32000,
-          "logprob": -13.4140625,
+          "logprob": -13.3984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7890625,
+          "logprob": -11.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6328125,
+          "logprob": -11.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.4296875,
+          "logprob": -14.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.53125,
+          "logprob": -13.5390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.515625,
+          "logprob": -14.5078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7265625,
+          "logprob": -12.7109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.609375,
+          "logprob": -11.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.171875,
+          "logprob": -13.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.109375,
+          "logprob": -13.09375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8828125,
+          "logprob": -12.875,
           "text": "<image>"
         },
         {
@@ -31260,37 +16466,37 @@
         },
         {
           "id": 32000,
-          "logprob": -12.671875,
+          "logprob": -12.6484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.7109375,
+          "logprob": -13.6953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4296875,
+          "logprob": -12.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.296875,
+          "logprob": -12.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1796875,
+          "logprob": -13.109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2421875,
+          "logprob": -12.1796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.828125,
+          "logprob": -14.875,
           "text": "<image>"
         },
         {
@@ -31300,7 +16506,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.3359375,
+          "logprob": -11.34375,
           "text": "<image>"
         },
         {
@@ -31320,67 +16526,7 @@
         },
         {
           "id": 32000,
-          "logprob": -15.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0,
+          "logprob": -15.0859375,
           "text": "<image>"
         },
         {
@@ -31390,57 +16536,117 @@
         },
         {
           "id": 32000,
-          "logprob": -16.21875,
+          "logprob": -11.875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6015625,
+          "logprob": -13.25,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.796875,
+          "logprob": -13.5078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7265625,
+          "logprob": -13.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9453125,
+          "logprob": -14.265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9765625,
+          "logprob": -14.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.71875,
+          "logprob": -13.2421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.6953125,
+          "logprob": -12.28125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3359375,
+          "logprob": -13.8046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3203125,
+          "logprob": -12.703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6328125,
+          "logprob": -12.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.609375,
           "text": "<image>"
         },
         {
@@ -31450,17 +16656,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1875,
+          "logprob": -14.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.046875,
+          "logprob": -12.0234375,
           "text": "<image>"
         },
         {
@@ -31475,17 +16681,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.8828125,
+          "logprob": -12.890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.296875,
+          "logprob": -12.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2109375,
+          "logprob": -12.21875,
           "text": "<image>"
         },
         {
@@ -31495,27 +16701,27 @@
         },
         {
           "id": 32000,
-          "logprob": -15.1875,
+          "logprob": -15.1953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5390625,
+          "logprob": -11.546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.421875,
+          "logprob": -14.4375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2890625,
+          "logprob": -12.265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2265625,
+          "logprob": -12.21875,
           "text": "<image>"
         },
         {
@@ -31530,12 +16736,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.5859375,
+          "logprob": -13.5390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.859375,
+          "logprob": -11.8515625,
           "text": "<image>"
         },
         {
@@ -31545,12 +16751,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.1015625,
+          "logprob": -13.046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.9453125,
+          "logprob": -13.953125,
           "text": "<image>"
         },
         {
@@ -31565,47 +16771,47 @@
         },
         {
           "id": 32000,
-          "logprob": -12.734375,
+          "logprob": -12.7109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.203125,
+          "logprob": -12.1796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.59375,
+          "logprob": -12.578125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3984375,
+          "logprob": -11.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5,
+          "logprob": -13.4609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.765625,
+          "logprob": -11.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2265625,
+          "logprob": -13.21875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7578125,
+          "logprob": -11.75,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.3515625,
+          "logprob": -15.34375,
           "text": "<image>"
         },
         {
@@ -31620,22 +16826,42 @@
         },
         {
           "id": 32000,
-          "logprob": -13.3671875,
+          "logprob": -13.3515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.90625,
+          "logprob": -11.8828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5625,
+          "logprob": -12.578125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3203125,
+          "logprob": -13.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.125,
           "text": "<image>"
         },
         {
@@ -31645,27 +16871,7 @@
         },
         {
           "id": 32000,
-          "logprob": -10.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9921875,
+          "logprob": -11.984375,
           "text": "<image>"
         },
         {
@@ -31680,27 +16886,27 @@
         },
         {
           "id": 32000,
-          "logprob": -12.9765625,
+          "logprob": -12.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4921875,
+          "logprob": -12.484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0625,
+          "logprob": -12.046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1796875,
+          "logprob": -12.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3359375,
+          "logprob": -13.296875,
           "text": "<image>"
         },
         {
@@ -31710,32 +16916,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.375,
+          "logprob": -12.3671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.984375,
+          "logprob": -13.1328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9453125,
+          "logprob": -13.625,
           "text": "<image>"
         },
         {
@@ -31745,87 +16936,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.3203125,
+          "logprob": -11.75,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3125,
+          "logprob": -11.9140625,
           "text": "<image>"
         },
         {
@@ -31835,12 +16951,102 @@
         },
         {
           "id": 32000,
-          "logprob": -11.3984375,
+          "logprob": -12.3359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.84375,
+          "logprob": -12.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8359375,
           "text": "<image>"
         },
         {
@@ -31850,52 +17056,52 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6328125,
+          "logprob": -12.6171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.875,
+          "logprob": -13.84375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.109375,
+          "logprob": -12.09375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1171875,
+          "logprob": -12.140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4921875,
+          "logprob": -11.5078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.2890625,
+          "logprob": -11.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.15625,
+          "logprob": -12.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.59375,
+          "logprob": -13.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8046875,
+          "logprob": -11.7890625,
           "text": "<image>"
         },
         {
@@ -31905,22 +17111,22 @@
         },
         {
           "id": 32000,
-          "logprob": -13.1015625,
+          "logprob": -13.0625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2265625,
+          "logprob": -13.1875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2109375,
+          "logprob": -13.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4609375,
+          "logprob": -12.46875,
           "text": "<image>"
         },
         {
@@ -31930,12 +17136,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
+          "logprob": -12.6953125,
           "text": "<image>"
         },
         {
@@ -31943,6 +17144,11 @@
           "logprob": -11.6875,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -11.6953125,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -11.7109375,
@@ -31950,27 +17156,27 @@
         },
         {
           "id": 32000,
-          "logprob": -15.390625,
+          "logprob": -15.3359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.953125,
+          "logprob": -11.9453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.140625,
+          "logprob": -16.125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.4921875,
+          "logprob": -15.4609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.9296875,
+          "logprob": -13.921875,
           "text": "<image>"
         },
         {
@@ -31985,7 +17191,7 @@
         },
         {
           "id": 32000,
-          "logprob": -15.984375,
+          "logprob": -16.0,
           "text": "<image>"
         },
         {
@@ -31995,37 +17201,37 @@
         },
         {
           "id": 32000,
-          "logprob": -12.8671875,
+          "logprob": -12.828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4921875,
+          "logprob": -12.5078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7421875,
+          "logprob": -11.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1875,
+          "logprob": -14.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3515625,
+          "logprob": -11.34375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.71875,
+          "logprob": -11.7265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0625,
+          "logprob": -13.046875,
           "text": "<image>"
         },
         {
@@ -32035,12 +17241,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.421875,
+          "logprob": -12.4375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.2734375,
+          "logprob": -14.265625,
           "text": "<image>"
         },
         {
@@ -32050,77 +17256,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.390625,
+          "logprob": -12.6953125,
           "text": "<image>"
         },
         {
@@ -32130,27 +17266,97 @@
         },
         {
           "id": 32000,
-          "logprob": -13.078125,
+          "logprob": -12.984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4296875,
+          "logprob": -13.2890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.828125,
+          "logprob": -13.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8046875,
+          "logprob": -11.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6484375,
+          "logprob": -10.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.625,
           "text": "<image>"
         },
         {
@@ -32165,27 +17371,27 @@
         },
         {
           "id": 32000,
-          "logprob": -14.921875,
+          "logprob": -14.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.578125,
+          "logprob": -12.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.78125,
+          "logprob": -12.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3984375,
+          "logprob": -11.375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.0546875,
+          "logprob": -14.03125,
           "text": "<image>"
         },
         {
@@ -32205,12 +17411,12 @@
         },
         {
           "id": 32000,
-          "logprob": -14.5234375,
+          "logprob": -14.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.625,
+          "logprob": -12.609375,
           "text": "<image>"
         },
         {
@@ -32220,12 +17426,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.6015625,
+          "logprob": -13.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.28125,
+          "logprob": -13.2109375,
           "text": "<image>"
         },
         {
@@ -32245,17 +17451,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.0,
+          "logprob": -12.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6640625,
+          "logprob": -12.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.46875,
+          "logprob": -11.484375,
           "text": "<image>"
         },
         {
@@ -32265,12 +17471,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8828125,
+          "logprob": -11.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1015625,
+          "logprob": -13.078125,
           "text": "<image>"
         },
         {
@@ -32285,7 +17491,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.5234375,
+          "logprob": -11.546875,
           "text": "<image>"
         },
         {
@@ -32300,7 +17506,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.4921875,
+          "logprob": -12.5,
           "text": "<image>"
         },
         {
@@ -32315,122 +17521,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.7890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1484375,
+          "logprob": -12.078125,
           "text": "<image>"
         },
         {
@@ -32440,132 +17531,102 @@
         },
         {
           "id": 32000,
-          "logprob": -13.3828125,
+          "logprob": -13.2265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3203125,
+          "logprob": -14.8046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.34375,
+          "logprob": -12.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -11.8671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.140625,
+          "logprob": -13.1484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9921875,
+          "logprob": -12.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.15625,
+          "logprob": -10.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.84375,
+          "logprob": -14.6875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.21875,
+          "logprob": -15.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.46875,
+          "logprob": -15.453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1484375,
+          "logprob": -11.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.515625,
+          "logprob": -14.484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.84375,
+          "logprob": -12.4375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.015625,
+          "logprob": -16.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.28125,
+          "logprob": -14.875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6015625,
+          "logprob": -16.109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3984375,
+          "logprob": -13.2890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.375,
+          "logprob": -12.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5625,
+          "logprob": -11.8359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -9.9375,
+          "logprob": -12.9921875,
           "text": "<image>"
         },
         {
@@ -32575,17 +17636,162 @@
         },
         {
           "id": 32000,
-          "logprob": -12.203125,
+          "logprob": -12.1484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.2890625,
+          "logprob": -13.84375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1953125,
+          "logprob": -13.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -9.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.15625,
           "text": "<image>"
         },
         {
@@ -32598,21 +17804,6 @@
           "logprob": -12.28125,
           "text": "<image>"
         },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
         {
           "id": 32000,
           "logprob": -11.8984375,
@@ -32620,52 +17811,67 @@
         },
         {
           "id": 32000,
-          "logprob": -10.5859375,
+          "logprob": -14.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9921875,
+          "logprob": -14.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9921875,
+          "logprob": -11.8671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1015625,
+          "logprob": -10.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.5390625,
+          "logprob": -13.0,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.2578125,
+          "logprob": -12.8984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1171875,
+          "logprob": -12.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2421875,
+          "logprob": -15.765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5,
+          "logprob": -15.203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7265625,
+          "logprob": -14.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8359375,
           "text": "<image>"
         },
         {
@@ -32675,7 +17881,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.0,
+          "logprob": -11.0078125,
           "text": "<image>"
         },
         {
@@ -32685,22 +17891,22 @@
         },
         {
           "id": 32000,
-          "logprob": -13.28125,
+          "logprob": -13.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7734375,
+          "logprob": -11.7578125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0625,
+          "logprob": -13.09375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.671875,
+          "logprob": -13.6796875,
           "text": "<image>"
         },
         {
@@ -32710,7 +17916,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.3828125,
+          "logprob": -12.421875,
           "text": "<image>"
         },
         {
@@ -32720,22 +17926,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.25,
+          "logprob": -12.2890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.9453125,
+          "logprob": -11.921875,
           "text": "<image>"
         },
         {
@@ -32745,7 +17941,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.9375,
+          "logprob": -15.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
           "text": "<image>"
         },
         {
@@ -32755,37 +17961,37 @@
         },
         {
           "id": 32000,
-          "logprob": -11.3203125,
+          "logprob": -11.328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4921875,
+          "logprob": -11.5,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.359375,
+          "logprob": -11.3671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3359375,
+          "logprob": -11.3515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0546875,
+          "logprob": -12.0625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.359375,
+          "logprob": -12.3359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -9.6953125,
+          "logprob": -9.7109375,
           "text": "<image>"
         },
         {
@@ -32795,27 +18001,27 @@
         },
         {
           "id": 32000,
-          "logprob": -14.3203125,
+          "logprob": -14.2421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9609375,
+          "logprob": -11.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0859375,
+          "logprob": -12.09375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2109375,
+          "logprob": -12.1796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4921875,
+          "logprob": -11.5234375,
           "text": "<image>"
         },
         {
@@ -32830,7 +18036,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.0546875,
           "text": "<image>"
         },
         {
@@ -32845,22 +18051,22 @@
         },
         {
           "id": 32000,
-          "logprob": -11.9921875,
+          "logprob": -12.0,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2265625,
+          "logprob": -12.1953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9921875,
+          "logprob": -12.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6796875,
+          "logprob": -12.7109375,
           "text": "<image>"
         },
         {
@@ -32870,7 +18076,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.5703125,
+          "logprob": -13.5546875,
           "text": "<image>"
         },
         {
@@ -32885,17 +18091,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.9375,
+          "logprob": -11.953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.9453125,
+          "logprob": -13.8046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.984375,
+          "logprob": -11.9921875,
           "text": "<image>"
         },
         {
@@ -32905,17 +18111,17 @@
         },
         {
           "id": 32000,
-          "logprob": -10.03125,
+          "logprob": -10.015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7265625,
+          "logprob": -11.71875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.7890625,
+          "logprob": -13.8125,
           "text": "<image>"
         },
         {
@@ -32925,42 +18131,42 @@
         },
         {
           "id": 32000,
-          "logprob": -12.671875,
+          "logprob": -12.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.59375,
+          "logprob": -12.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1171875,
+          "logprob": -12.1015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4609375,
+          "logprob": -12.453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3046875,
+          "logprob": -12.2890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4765625,
+          "logprob": -12.46875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6328125,
+          "logprob": -12.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.03125,
+          "logprob": -12.015625,
           "text": "<image>"
         },
         {
@@ -32970,132 +18176,32 @@
         },
         {
           "id": 32000,
-          "logprob": -14.5078125,
+          "logprob": -14.421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2265625,
+          "logprob": -13.25,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.546875,
+          "logprob": -16.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.4765625,
+          "logprob": -14.484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.71875,
+          "logprob": -13.7421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.7890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.671875,
+          "logprob": -13.46875,
           "text": "<image>"
         },
         {
@@ -33105,57 +18211,97 @@
         },
         {
           "id": 32000,
-          "logprob": -13.40625,
+          "logprob": -12.8671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.265625,
+          "logprob": -12.4921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.015625,
+          "logprob": -13.2265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5390625,
+          "logprob": -13.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.0234375,
+          "logprob": -12.4375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1796875,
+          "logprob": -10.765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6875,
+          "logprob": -13.109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.9453125,
+          "logprob": -12.8515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6875,
+          "logprob": -14.7421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.78125,
+          "logprob": -12.84375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.9609375,
+          "logprob": -12.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6484375,
           "text": "<image>"
         },
         {
@@ -33165,27 +18311,87 @@
         },
         {
           "id": 32000,
-          "logprob": -16.140625,
+          "logprob": -13.4296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4609375,
+          "logprob": -13.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.7265625,
+          "logprob": -12.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.609375,
+          "logprob": -11.5234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.234375,
+          "logprob": -15.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.25,
           "text": "<image>"
         },
         {
@@ -33195,7 +18401,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.4921875,
+          "logprob": -13.5,
           "text": "<image>"
         },
         {
@@ -33220,12 +18426,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.15625,
+          "logprob": -13.1484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -17.046875,
+          "logprob": -17.0625,
           "text": "<image>"
         },
         {
@@ -33235,12 +18441,12 @@
         },
         {
           "id": 32000,
-          "logprob": -14.265625,
+          "logprob": -14.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6328125,
+          "logprob": -12.625,
           "text": "<image>"
         },
         {
@@ -33250,92 +18456,37 @@
         },
         {
           "id": 32000,
-          "logprob": -13.9375,
+          "logprob": -13.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0390625,
+          "logprob": -13.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.46875,
+          "logprob": -10.5,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1640625,
+          "logprob": -13.1953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.59375,
+          "logprob": -13.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.390625,
+          "logprob": -13.3828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8125,
+          "logprob": -14.0625,
           "text": "<image>"
         },
         {
@@ -33343,6 +18494,61 @@
           "logprob": -13.5234375,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -14.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.53125,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -11.4140625,
@@ -33350,17 +18556,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.46875,
+          "logprob": -11.4609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.09375,
+          "logprob": -13.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.34375,
+          "logprob": -12.3359375,
           "text": "<image>"
         },
         {
@@ -33370,32 +18576,32 @@
         },
         {
           "id": 32000,
-          "logprob": -12.7578125,
+          "logprob": -12.75,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.8359375,
+          "logprob": -14.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0859375,
+          "logprob": -13.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1640625,
+          "logprob": -14.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0625,
+          "logprob": -13.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.5078125,
+          "logprob": -14.4921875,
           "text": "<image>"
         },
         {
@@ -33405,7 +18611,7 @@
         },
         {
           "id": 32000,
-          "logprob": -14.6953125,
+          "logprob": -14.703125,
           "text": "<image>"
         },
         {
@@ -33415,32 +18621,32 @@
         },
         {
           "id": 32000,
-          "logprob": -16.328125,
+          "logprob": -16.3125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.9921875,
+          "logprob": -14.9765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.8515625,
+          "logprob": -13.8046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.40625,
+          "logprob": -15.375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4296875,
+          "logprob": -13.3984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.5859375,
+          "logprob": -14.578125,
           "text": "<image>"
         },
         {
@@ -33455,7 +18661,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.3359375,
+          "logprob": -13.328125,
           "text": "<image>"
         },
         {
@@ -33470,37 +18676,52 @@
         },
         {
           "id": 32000,
-          "logprob": -16.890625,
+          "logprob": -16.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.328125,
+          "logprob": -13.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.78125,
+          "logprob": -13.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.34375,
+          "logprob": -13.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4921875,
+          "logprob": -13.4765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4296875,
+          "logprob": -13.390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.28125,
+          "logprob": -13.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5,
           "text": "<image>"
         },
         {
@@ -33510,7 +18731,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.75,
+          "logprob": -13.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6796875,
           "text": "<image>"
         },
         {
@@ -33520,32 +18751,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.328125,
+          "logprob": -13.3203125,
           "text": "<image>"
         },
         {
@@ -33555,47 +18761,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.640625,
+          "logprob": -13.6171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.15625,
+          "logprob": -15.140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0078125,
+          "logprob": -13.21875,
           "text": "<image>"
         },
         {
@@ -33605,27 +18781,57 @@
         },
         {
           "id": 32000,
-          "logprob": -11.625,
+          "logprob": -12.0234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8671875,
+          "logprob": -11.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.578125,
+          "logprob": -11.3125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8125,
+          "logprob": -10.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.9453125,
+          "logprob": -13.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.9765625,
           "text": "<image>"
         },
         {
@@ -33635,17 +18841,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.9765625,
+          "logprob": -12.984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.828125,
+          "logprob": -13.8359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7265625,
+          "logprob": -12.75,
           "text": "<image>"
         },
         {
@@ -33660,12 +18866,12 @@
         },
         {
           "id": 32000,
-          "logprob": -14.7578125,
+          "logprob": -14.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.5234375,
+          "logprob": -14.5078125,
           "text": "<image>"
         },
         {
@@ -33675,12 +18881,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.0390625,
+          "logprob": -13.046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4453125,
+          "logprob": -13.453125,
           "text": "<image>"
         },
         {
@@ -33695,17 +18901,17 @@
         },
         {
           "id": 32000,
-          "logprob": -14.0,
+          "logprob": -13.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5390625,
+          "logprob": -13.4921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0859375,
+          "logprob": -11.9765625,
           "text": "<image>"
         },
         {
@@ -33715,17 +18921,37 @@
         },
         {
           "id": 32000,
-          "logprob": -15.953125,
+          "logprob": -15.9609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.2265625,
+          "logprob": -14.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1484375,
+          "logprob": -13.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.640625,
           "text": "<image>"
         },
         {
@@ -33735,32 +18961,47 @@
         },
         {
           "id": 32000,
-          "logprob": -12.6875,
+          "logprob": -13.625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.796875,
+          "logprob": -12.890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.65625,
+          "logprob": -12.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6796875,
+          "logprob": -12.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.65625,
+          "logprob": -12.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.90625,
+          "logprob": -14.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0078125,
           "text": "<image>"
         },
         {
@@ -33770,52 +19011,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.6171875,
+          "logprob": -15.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7890625,
+          "logprob": -12.921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.375,
+          "logprob": -12.40625,
           "text": "<image>"
         },
         {
@@ -33830,27 +19036,27 @@
         },
         {
           "id": 32000,
-          "logprob": -11.375,
+          "logprob": -11.3671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0625,
+          "logprob": -11.0703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3046875,
+          "logprob": -11.28125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0390625,
+          "logprob": -11.046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3515625,
+          "logprob": -13.3203125,
           "text": "<image>"
         },
         {
@@ -33875,12 +19081,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.8359375,
+          "logprob": -13.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0234375,
+          "logprob": -12.0859375,
           "text": "<image>"
         },
         {
@@ -33890,12 +19096,7 @@
         },
         {
           "id": 32000,
-          "logprob": -14.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0546875,
+          "logprob": -14.0703125,
           "text": "<image>"
         },
         {
@@ -33905,7 +19106,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.0625,
+          "logprob": -14.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0390625,
           "text": "<image>"
         },
         {
@@ -33915,42 +19121,42 @@
         },
         {
           "id": 32000,
-          "logprob": -14.46875,
+          "logprob": -14.453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.703125,
+          "logprob": -14.6875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.296875,
+          "logprob": -14.3046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.8828125,
+          "logprob": -14.8671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1796875,
+          "logprob": -14.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2265625,
+          "logprob": -13.203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1171875,
+          "logprob": -13.109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9375,
+          "logprob": -11.8515625,
           "text": "<image>"
         },
         {
@@ -33965,12 +19171,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.71875,
+          "logprob": -12.7109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.734375,
+          "logprob": -13.7109375,
           "text": "<image>"
         },
         {
@@ -33980,32 +19186,32 @@
         },
         {
           "id": 32000,
-          "logprob": -12.46875,
+          "logprob": -12.4765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.15625,
+          "logprob": -12.140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.375,
+          "logprob": -12.3828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8984375,
+          "logprob": -12.875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.6953125,
+          "logprob": -13.6484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0625,
+          "logprob": -13.0390625,
           "text": "<image>"
         },
         {
@@ -34015,12 +19221,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.6796875,
+          "logprob": -12.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3515625,
+          "logprob": -12.34375,
           "text": "<image>"
         },
         {
@@ -34030,37 +19236,37 @@
         },
         {
           "id": 32000,
-          "logprob": -12.46875,
+          "logprob": -12.453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.03125,
+          "logprob": -12.015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.96875,
+          "logprob": -11.9765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.4609375,
+          "logprob": -14.46875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.328125,
+          "logprob": -13.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6484375,
+          "logprob": -12.71875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.65625,
+          "logprob": -10.6484375,
           "text": "<image>"
         },
         {
@@ -34080,22 +19286,22 @@
         },
         {
           "id": 32000,
-          "logprob": -12.6328125,
+          "logprob": -12.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.265625,
+          "logprob": -11.2578125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.734375,
+          "logprob": -11.7265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.84375,
+          "logprob": -10.8515625,
           "text": "<image>"
         },
         {
@@ -34105,32 +19311,32 @@
         },
         {
           "id": 32000,
-          "logprob": -13.109375,
+          "logprob": -13.140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7421875,
+          "logprob": -12.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.09375,
+          "logprob": -12.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.109375,
+          "logprob": -13.1328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.671875,
+          "logprob": -12.65625,
           "text": "<image>"
         },
         {
@@ -34140,27 +19346,27 @@
         },
         {
           "id": 32000,
-          "logprob": -12.6796875,
+          "logprob": -12.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.109375,
+          "logprob": -12.0859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.625,
+          "logprob": -13.609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8515625,
+          "logprob": -12.8359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3046875,
+          "logprob": -12.3125,
           "text": "<image>"
         },
         {
@@ -34170,12 +19376,12 @@
         },
         {
           "id": 32000,
-          "logprob": -14.0703125,
+          "logprob": -14.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.0,
+          "logprob": -13.984375,
           "text": "<image>"
         },
         {
@@ -34185,7 +19391,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.2421875,
+          "logprob": -13.2109375,
           "text": "<image>"
         },
         {
@@ -34195,42 +19401,42 @@
         },
         {
           "id": 32000,
-          "logprob": -13.7109375,
+          "logprob": -13.703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6015625,
+          "logprob": -12.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.796875,
+          "logprob": -15.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.90625,
+          "logprob": -11.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.375,
+          "logprob": -12.3671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4765625,
+          "logprob": -11.46875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.015625,
+          "logprob": -12.0,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.328125,
+          "logprob": -11.3359375,
           "text": "<image>"
         },
         {
@@ -34245,17 +19451,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.8203125,
+          "logprob": -13.828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7578125,
+          "logprob": -11.75,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.484375,
+          "logprob": -11.46875,
           "text": "<image>"
         },
         {
@@ -34270,17 +19476,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.640625,
+          "logprob": -11.625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5859375,
+          "logprob": -11.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.2578125,
+          "logprob": -15.2265625,
           "text": "<image>"
         },
         {
@@ -34290,17 +19496,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.0078125,
+          "logprob": -13.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.3828125,
+          "logprob": -15.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.203125,
+          "logprob": -11.1875,
           "text": "<image>"
         },
         {
@@ -34310,7 +19516,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.0703125,
+          "logprob": -11.0625,
           "text": "<image>"
         },
         {
@@ -34320,7 +19526,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.4453125,
+          "logprob": -13.453125,
           "text": "<image>"
         },
         {
@@ -34330,27 +19536,27 @@
         },
         {
           "id": 32000,
-          "logprob": -11.5078125,
+          "logprob": -11.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.015625,
+          "logprob": -12.0078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.15625,
+          "logprob": -11.1875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8359375,
+          "logprob": -11.890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.359375,
+          "logprob": -11.3125,
           "text": "<image>"
         },
         {
@@ -34365,22 +19571,22 @@
         },
         {
           "id": 32000,
-          "logprob": -12.4453125,
+          "logprob": -12.4921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.015625,
+          "logprob": -12.0703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.375,
+          "logprob": -13.3671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2734375,
+          "logprob": -13.28125,
           "text": "<image>"
         },
         {
@@ -34390,22 +19596,22 @@
         },
         {
           "id": 32000,
-          "logprob": -13.046875,
+          "logprob": -13.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0859375,
+          "logprob": -13.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1640625,
+          "logprob": -12.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4765625,
+          "logprob": -13.484375,
           "text": "<image>"
         },
         {
@@ -34415,7 +19621,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.7265625,
+          "logprob": -13.703125,
           "text": "<image>"
         },
         {
@@ -34425,262 +19631,17 @@
         },
         {
           "id": 32000,
-          "logprob": -14.3359375,
+          "logprob": -14.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.71875,
+          "logprob": -12.7109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.640625,
+          "logprob": -14.28125,
           "text": "<image>"
         },
         {
@@ -34690,52 +19651,57 @@
         },
         {
           "id": 32000,
-          "logprob": -13.453125,
+          "logprob": -13.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.4375,
+          "logprob": -15.203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1171875,
+          "logprob": -13.28125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4296875,
+          "logprob": -12.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.515625,
+          "logprob": -14.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.21875,
+          "logprob": -13.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8671875,
+          "logprob": -12.1171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8515625,
+          "logprob": -14.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.8671875,
+          "logprob": -14.0625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.59375,
+          "logprob": -12.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.953125,
           "text": "<image>"
         },
         {
@@ -34745,12 +19711,57 @@
         },
         {
           "id": 32000,
-          "logprob": -12.1640625,
+          "logprob": -11.6953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2265625,
+          "logprob": -12.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5234375,
           "text": "<image>"
         },
         {
@@ -34760,32 +19771,27 @@
         },
         {
           "id": 32000,
-          "logprob": -12.1015625,
+          "logprob": -11.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0234375,
+          "logprob": -11.2578125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4140625,
+          "logprob": -13.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.828125,
+          "logprob": -11.921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1953125,
+          "logprob": -13.03125,
           "text": "<image>"
         },
         {
@@ -34795,7 +19801,207 @@
         },
         {
           "id": 32000,
-          "logprob": -11.1328125,
+          "logprob": -14.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.4296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.140625,
           "text": "<image>"
         },
         {
@@ -34820,7 +20026,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.1015625,
+          "logprob": -11.1171875,
           "text": "<image>"
         },
         {
@@ -34830,7 +20036,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.15625,
+          "logprob": -11.1484375,
           "text": "<image>"
         },
         {
@@ -34840,17 +20046,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.4296875,
+          "logprob": -11.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.765625,
+          "logprob": -12.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.09375,
+          "logprob": -13.0859375,
           "text": "<image>"
         },
         {
@@ -34860,22 +20066,22 @@
         },
         {
           "id": 32000,
-          "logprob": -12.96875,
+          "logprob": -12.9609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.328125,
+          "logprob": -12.34375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8359375,
+          "logprob": -12.84375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.9609375,
+          "logprob": -13.9765625,
           "text": "<image>"
         },
         {
@@ -34885,7 +20091,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.46875,
+          "logprob": -13.484375,
           "text": "<image>"
         },
         {
@@ -34895,22 +20101,22 @@
         },
         {
           "id": 32000,
-          "logprob": -13.59375,
+          "logprob": -13.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.40625,
+          "logprob": -13.3828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.234375,
+          "logprob": -14.25,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.265625,
+          "logprob": -14.2578125,
           "text": "<image>"
         },
         {
@@ -34920,127 +20126,127 @@
         },
         {
           "id": 32000,
-          "logprob": -13.65625,
+          "logprob": -13.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.2734375,
+          "logprob": -14.2578125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.09375,
+          "logprob": -13.8671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.890625,
+          "logprob": -12.921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.640625,
+          "logprob": -15.6484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0234375,
+          "logprob": -13.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3828125,
+          "logprob": -12.34375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.71875,
+          "logprob": -12.7265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.5234375,
+          "logprob": -14.515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7734375,
+          "logprob": -12.75,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3203125,
+          "logprob": -12.265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.609375,
+          "logprob": -13.546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1640625,
+          "logprob": -13.1328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.203125,
+          "logprob": -12.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1875,
+          "logprob": -12.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.59375,
+          "logprob": -11.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6875,
+          "logprob": -11.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1796875,
+          "logprob": -12.15625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6484375,
+          "logprob": -11.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.421875,
+          "logprob": -11.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.34375,
+          "logprob": -11.390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1015625,
+          "logprob": -11.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9765625,
+          "logprob": -11.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.015625,
+          "logprob": -12.0078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0,
+          "logprob": -12.984375,
           "text": "<image>"
         },
         {
@@ -35050,7 +20256,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.09375,
           "text": "<image>"
         },
         {
@@ -35070,12 +20276,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.65625,
+          "logprob": -11.6484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.84375,
+          "logprob": -12.8125,
           "text": "<image>"
         },
         {
@@ -35085,17 +20291,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.6953125,
+          "logprob": -11.6796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.578125,
+          "logprob": -11.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.6796875,
+          "logprob": -13.6953125,
           "text": "<image>"
         },
         {
@@ -35110,7 +20316,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0078125,
+          "logprob": -12.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.296875,
           "text": "<image>"
         },
         {
@@ -35120,17 +20331,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.234375,
+          "logprob": -13.84375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8671875,
+          "logprob": -12.875,
           "text": "<image>"
         },
         {
@@ -35140,17 +20346,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.5078125,
+          "logprob": -13.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.953125,
+          "logprob": -13.9609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5078125,
+          "logprob": -12.46875,
           "text": "<image>"
         },
         {
@@ -35160,232 +20366,17 @@
         },
         {
           "id": 32000,
-          "logprob": -14.25,
+          "logprob": -14.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9140625,
+          "logprob": -12.921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4375,
+          "logprob": -14.2578125,
           "text": "<image>"
         },
         {
@@ -35395,62 +20386,62 @@
         },
         {
           "id": 32000,
-          "logprob": -13.78125,
+          "logprob": -14.3984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5703125,
+          "logprob": -15.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9765625,
+          "logprob": -13.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4765625,
+          "logprob": -12.3515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.125,
+          "logprob": -13.46875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0859375,
+          "logprob": -11.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9296875,
+          "logprob": -12.2890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.90625,
+          "logprob": -14.3515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0625,
+          "logprob": -13.140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0234375,
+          "logprob": -13.0234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.6953125,
+          "logprob": -12.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.890625,
+          "logprob": -11.8125,
           "text": "<image>"
         },
         {
@@ -35458,6 +20449,31 @@
           "logprob": -12.125,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -11.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.640625,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -12.4375,
@@ -35465,67 +20481,257 @@
         },
         {
           "id": 32000,
-          "logprob": -12.375,
+          "logprob": -11.46875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.28125,
+          "logprob": -11.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4921875,
+          "logprob": -12.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1640625,
+          "logprob": -11.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1953125,
+          "logprob": -11.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.4140625,
+          "logprob": -12.4453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6171875,
+          "logprob": -11.34375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.921875,
+          "logprob": -12.1015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6328125,
+          "logprob": -11.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2421875,
+          "logprob": -14.6015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0234375,
+          "logprob": -12.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6953125,
+          "logprob": -11.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1484375,
+          "logprob": -11.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1328125,
           "text": "<image>"
         },
         {
@@ -35535,27 +20741,27 @@
         },
         {
           "id": 32000,
-          "logprob": -11.7265625,
+          "logprob": -11.71875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6796875,
+          "logprob": -11.6875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4921875,
+          "logprob": -12.484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.875,
+          "logprob": -11.8828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5390625,
+          "logprob": -12.53125,
           "text": "<image>"
         },
         {
@@ -35570,27 +20776,27 @@
         },
         {
           "id": 32000,
-          "logprob": -13.71875,
+          "logprob": -13.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.78125,
+          "logprob": -14.8046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.578125,
+          "logprob": -11.609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5859375,
+          "logprob": -13.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.390625,
+          "logprob": -12.421875,
           "text": "<image>"
         },
         {
@@ -35600,17 +20806,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.65625,
+          "logprob": -13.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5,
+          "logprob": -12.5078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.109375,
+          "logprob": -13.125,
           "text": "<image>"
         },
         {
@@ -35620,37 +20826,37 @@
         },
         {
           "id": 32000,
-          "logprob": -12.2578125,
+          "logprob": -12.2890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7421875,
+          "logprob": -12.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1640625,
+          "logprob": -13.15625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9375,
+          "logprob": -11.9453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.265625,
+          "logprob": -12.28125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1640625,
+          "logprob": -12.1328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.8671875,
+          "logprob": -13.8203125,
           "text": "<image>"
         },
         {
@@ -35660,57 +20866,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.046875,
+          "logprob": -13.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.984375,
+          "logprob": -15.6953125,
           "text": "<image>"
         },
         {
@@ -35720,12 +20881,47 @@
         },
         {
           "id": 32000,
-          "logprob": -11.3984375,
+          "logprob": -14.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.796875,
+          "logprob": -13.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.234375,
           "text": "<image>"
         },
         {
@@ -35735,37 +20931,47 @@
         },
         {
           "id": 32000,
-          "logprob": -11.4296875,
+          "logprob": -11.828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9921875,
+          "logprob": -11.453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.59375,
+          "logprob": -11.453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8359375,
+          "logprob": -11.984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7421875,
+          "logprob": -11.609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.125,
+          "logprob": -11.8046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.875,
+          "logprob": -11.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8515625,
           "text": "<image>"
         },
         {
@@ -35775,27 +20981,27 @@
         },
         {
           "id": 32000,
-          "logprob": -10.796875,
+          "logprob": -10.8046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4765625,
+          "logprob": -11.578125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1484375,
+          "logprob": -11.15625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8046875,
+          "logprob": -12.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3828125,
+          "logprob": -11.3671875,
           "text": "<image>"
         },
         {
@@ -35815,7 +21021,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.5546875,
+          "logprob": -12.5625,
           "text": "<image>"
         },
         {
@@ -35825,367 +21031,57 @@
         },
         {
           "id": 32000,
-          "logprob": -10.40625,
+          "logprob": -10.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.59375,
+          "logprob": -12.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.03125,
+          "logprob": -12.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.03125,
+          "logprob": -12.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.65625,
+          "logprob": -12.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6328125,
+          "logprob": -11.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5,
+          "logprob": -13.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.078125,
+          "logprob": -12.0703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0234375,
+          "logprob": -13.015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.171875,
+          "logprob": -14.1484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.4609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.015625,
+          "logprob": -12.3671875,
           "text": "<image>"
         },
         {
@@ -36195,42 +21091,112 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0859375,
+          "logprob": -12.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.140625,
+          "logprob": -13.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.65625,
+          "logprob": -13.953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.890625,
+          "logprob": -12.4765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.59375,
+          "logprob": -14.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4375,
+          "logprob": -12.6015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4375,
+          "logprob": -13.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.109375,
+          "logprob": -15.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0546875,
           "text": "<image>"
         },
         {
@@ -36240,17 +21206,257 @@
         },
         {
           "id": 32000,
-          "logprob": -11.1015625,
+          "logprob": -11.9609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.203125,
+          "logprob": -14.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9921875,
+          "logprob": -12.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.1484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.96875,
           "text": "<image>"
         },
         {
@@ -36260,32 +21466,32 @@
         },
         {
           "id": 32000,
-          "logprob": -11.40625,
+          "logprob": -11.4296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.53125,
+          "logprob": -12.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.109375,
+          "logprob": -12.1015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3203125,
+          "logprob": -11.3125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.125,
+          "logprob": -11.1171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7578125,
+          "logprob": -11.75,
           "text": "<image>"
         },
         {
@@ -36300,7 +21506,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.109375,
+          "logprob": -11.1015625,
           "text": "<image>"
         },
         {
@@ -36320,67 +21526,67 @@
         },
         {
           "id": 32000,
-          "logprob": -12.609375,
+          "logprob": -12.6015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.65625,
+          "logprob": -12.6796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7890625,
+          "logprob": -12.8359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.28125,
+          "logprob": -12.296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4140625,
+          "logprob": -13.3671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.96875,
+          "logprob": -12.9765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1171875,
+          "logprob": -12.109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5390625,
+          "logprob": -12.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.609375,
+          "logprob": -12.6015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6484375,
+          "logprob": -12.625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.09375,
+          "logprob": -12.109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9921875,
+          "logprob": -13.0,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4453125,
+          "logprob": -13.4296875,
           "text": "<image>"
         },
         {
@@ -36390,17 +21596,17 @@
         },
         {
           "id": 32000,
-          "logprob": -14.7109375,
+          "logprob": -14.71875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.625,
+          "logprob": -13.6015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.765625,
+          "logprob": -13.7578125,
           "text": "<image>"
         },
         {
@@ -36410,37 +21616,37 @@
         },
         {
           "id": 32000,
-          "logprob": -14.53125,
+          "logprob": -14.5390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.34375,
+          "logprob": -13.3515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.921875,
+          "logprob": -12.9453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.359375,
+          "logprob": -11.3515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.078125,
+          "logprob": -12.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4609375,
+          "logprob": -11.421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.640625,
+          "logprob": -11.625,
           "text": "<image>"
         },
         {
@@ -36455,62 +21661,62 @@
         },
         {
           "id": 32000,
-          "logprob": -11.953125,
+          "logprob": -11.9453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0859375,
+          "logprob": -12.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.421875,
+          "logprob": -11.4296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5859375,
+          "logprob": -11.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0859375,
+          "logprob": -12.0703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.25,
+          "logprob": -11.2421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6171875,
+          "logprob": -11.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3359375,
+          "logprob": -11.3125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.46875,
+          "logprob": -11.453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.265625,
+          "logprob": -12.28125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.21875,
+          "logprob": -12.2109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1875,
+          "logprob": -12.203125,
           "text": "<image>"
         },
         {
@@ -36525,47 +21731,47 @@
         },
         {
           "id": 32000,
-          "logprob": -11.2421875,
+          "logprob": -11.25,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3046875,
+          "logprob": -12.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.03125,
+          "logprob": -11.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.203125,
+          "logprob": -11.1953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.125,
+          "logprob": -11.1171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9296875,
+          "logprob": -11.9375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9765625,
+          "logprob": -10.984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.453125,
+          "logprob": -11.390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.859375,
+          "logprob": -12.8828125,
           "text": "<image>"
         },
         {
@@ -36590,47 +21796,47 @@
         },
         {
           "id": 32000,
-          "logprob": -12.1328125,
+          "logprob": -12.1171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5,
+          "logprob": -12.515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8671875,
+          "logprob": -11.890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1640625,
+          "logprob": -13.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.765625,
+          "logprob": -15.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.609375,
+          "logprob": -11.6015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.34375,
+          "logprob": -13.328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.671875,
+          "logprob": -13.6484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.9140625,
+          "logprob": -14.875,
           "text": "<image>"
         },
         {
@@ -36640,57 +21846,87 @@
         },
         {
           "id": 32000,
-          "logprob": -13.53125,
+          "logprob": -13.5234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.25,
+          "logprob": -13.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6171875,
+          "logprob": -12.609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.8046875,
+          "logprob": -13.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.375,
+          "logprob": -14.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.171875,
+          "logprob": -12.1484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.828125,
+          "logprob": -11.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.875,
+          "logprob": -11.890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.484375,
+          "logprob": -11.4921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5625,
+          "logprob": -12.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.640625,
+          "logprob": -11.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.03125,
           "text": "<image>"
         },
         {
@@ -36698,36 +21934,6 @@
           "logprob": -11.7265625,
           "text": "<image>"
         },
-        {
-          "id": 32000,
-          "logprob": -11.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.734375,
-          "text": "<image>"
-        },
         {
           "id": 32000,
           "logprob": -11.84375,
@@ -36735,12 +21941,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0390625,
+          "logprob": -11.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.2578125,
+          "logprob": -11.2421875,
           "text": "<image>"
         },
         {
@@ -36775,22 +21981,22 @@
         },
         {
           "id": 32000,
-          "logprob": -11.0390625,
+          "logprob": -11.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1953125,
+          "logprob": -12.203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.6640625,
+          "logprob": -13.609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6640625,
+          "logprob": -12.65625,
           "text": "<image>"
         },
         {
@@ -36815,27 +22021,47 @@
         },
         {
           "id": 32000,
-          "logprob": -11.9140625,
+          "logprob": -11.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.34375,
+          "logprob": -11.3359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.578125,
+          "logprob": -11.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.953125,
+          "logprob": -11.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.296875,
+          "logprob": -11.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.875,
           "text": "<image>"
         },
         {
@@ -36845,27 +22071,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9765625,
+          "logprob": -12.046875,
           "text": "<image>"
         },
         {
@@ -36875,7 +22081,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.734375,
+          "logprob": -13.7265625,
           "text": "<image>"
         },
         {
@@ -36885,7 +22091,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.8359375,
+          "logprob": -13.8203125,
           "text": "<image>"
         },
         {
@@ -36900,37 +22106,37 @@
         },
         {
           "id": 32000,
-          "logprob": -14.5625,
+          "logprob": -14.546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.0234375,
+          "logprob": -14.015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5703125,
+          "logprob": -12.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8125,
+          "logprob": -11.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.7109375,
+          "logprob": -10.6953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6015625,
+          "logprob": -11.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8671875,
+          "logprob": -12.0234375,
           "text": "<image>"
         },
         {
@@ -36945,32 +22151,32 @@
         },
         {
           "id": 32000,
-          "logprob": -14.3359375,
+          "logprob": -14.34375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.90625,
+          "logprob": -11.8984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.296875,
+          "logprob": -13.4375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.984375,
+          "logprob": -11.9765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0625,
+          "logprob": -11.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4296875,
+          "logprob": -11.421875,
           "text": "<image>"
         },
         {
@@ -36985,27 +22191,27 @@
         },
         {
           "id": 32000,
-          "logprob": -11.28125,
+          "logprob": -11.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5546875,
+          "logprob": -11.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.671875,
+          "logprob": -11.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1171875,
+          "logprob": -12.125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5,
+          "logprob": -11.484375,
           "text": "<image>"
         },
         {
@@ -37015,27 +22221,27 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8984375,
+          "logprob": -11.875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2890625,
+          "logprob": -12.3125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6484375,
+          "logprob": -11.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8984375,
+          "logprob": -12.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.734375,
+          "logprob": -11.7421875,
           "text": "<image>"
         },
         {
@@ -37045,7 +22251,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.40625,
+          "logprob": -11.3984375,
           "text": "<image>"
         },
         {
@@ -37055,7 +22261,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.3515625,
+          "logprob": -11.34375,
           "text": "<image>"
         },
         {
@@ -37070,37 +22276,37 @@
         },
         {
           "id": 32000,
-          "logprob": -11.03125,
+          "logprob": -10.9609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.375,
+          "logprob": -13.265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6640625,
+          "logprob": -12.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.96875,
+          "logprob": -11.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.671875,
+          "logprob": -14.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.390625,
+          "logprob": -12.3828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.2890625,
+          "logprob": -14.3515625,
           "text": "<image>"
         },
         {
@@ -37110,32 +22316,32 @@
         },
         {
           "id": 32000,
-          "logprob": -15.1796875,
+          "logprob": -15.125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6875,
+          "logprob": -12.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7421875,
+          "logprob": -12.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.6484375,
+          "logprob": -14.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.546875,
+          "logprob": -13.5234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5703125,
+          "logprob": -12.5859375,
           "text": "<image>"
         },
         {
@@ -37145,32 +22351,17 @@
         },
         {
           "id": 32000,
-          "logprob": -14.390625,
+          "logprob": -14.3828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.96875,
+          "logprob": -12.9453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7421875,
+          "logprob": -12.609375,
           "text": "<image>"
         },
         {
@@ -37180,32 +22371,47 @@
         },
         {
           "id": 32000,
-          "logprob": -12.453125,
+          "logprob": -12.953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.8046875,
+          "logprob": -11.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.0078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.7890625,
+          "logprob": -12.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.3828125,
+          "logprob": -13.8828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.828125,
+          "logprob": -12.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8359375,
           "text": "<image>"
         },
         {
@@ -37215,12 +22421,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.0,
+          "logprob": -13.0078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1171875,
+          "logprob": -12.140625,
           "text": "<image>"
         },
         {
@@ -37230,17 +22436,17 @@
         },
         {
           "id": 32000,
-          "logprob": -14.015625,
+          "logprob": -13.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6328125,
+          "logprob": -11.625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4765625,
+          "logprob": -11.484375,
           "text": "<image>"
         },
         {
@@ -37250,7 +22456,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.5859375,
+          "logprob": -11.5703125,
           "text": "<image>"
         },
         {
@@ -37265,7 +22471,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0390625,
+          "logprob": -12.03125,
           "text": "<image>"
         },
         {
@@ -37275,7 +22481,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.4609375,
+          "logprob": -11.46875,
           "text": "<image>"
         },
         {
@@ -37285,7 +22491,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0625,
+          "logprob": -12.078125,
           "text": "<image>"
         },
         {
@@ -37295,142 +22501,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
+          "logprob": -11.296875,
           "text": "<image>"
         },
         {
@@ -37440,7 +22511,142 @@
         },
         {
           "id": 32000,
-          "logprob": -11.34375,
+          "logprob": -11.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.21875,
           "text": "<image>"
         },
         {
@@ -37450,52 +22656,52 @@
         },
         {
           "id": 32000,
-          "logprob": -11.3671875,
+          "logprob": -11.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0,
+          "logprob": -12.984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.1875,
+          "logprob": -15.15625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0,
+          "logprob": -11.9765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.75,
+          "logprob": -12.765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.515625,
+          "logprob": -13.4921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5703125,
+          "logprob": -11.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.703125,
+          "logprob": -11.6953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3671875,
+          "logprob": -11.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4921875,
+          "logprob": -11.5,
           "text": "<image>"
         },
         {
@@ -37505,12 +22711,12 @@
         },
         {
           "id": 32000,
-          "logprob": -14.5546875,
+          "logprob": -14.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.390625,
+          "logprob": -13.4140625,
           "text": "<image>"
         },
         {
@@ -37520,7 +22726,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.7265625,
+          "logprob": -11.71875,
           "text": "<image>"
         },
         {
@@ -37535,7 +22741,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.2421875,
+          "logprob": -13.2578125,
           "text": "<image>"
         },
         {
@@ -37555,27 +22761,27 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8203125,
+          "logprob": -11.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9296875,
+          "logprob": -11.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.796875,
+          "logprob": -11.765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1875,
+          "logprob": -11.1953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.359375,
+          "logprob": -13.3359375,
           "text": "<image>"
         },
         {
@@ -37585,7 +22791,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.6796875,
+          "logprob": -12.703125,
           "text": "<image>"
         },
         {
@@ -37600,27 +22806,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.5625,
+          "logprob": -13.6171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6484375,
+          "logprob": -13.328125,
           "text": "<image>"
         },
         {
@@ -37628,6 +22819,21 @@
           "logprob": -12.828125,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -13.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.796875,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -12.78125,
@@ -37635,42 +22841,42 @@
         },
         {
           "id": 32000,
-          "logprob": -12.578125,
+          "logprob": -12.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7578125,
+          "logprob": -12.765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.484375,
+          "logprob": -11.46875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5390625,
+          "logprob": -12.484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6015625,
+          "logprob": -11.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.640625,
+          "logprob": -11.6171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.140625,
+          "logprob": -12.1328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.65625,
+          "logprob": -12.59375,
           "text": "<image>"
         },
         {
@@ -37680,17 +22886,17 @@
         },
         {
           "id": 32000,
-          "logprob": -14.265625,
+          "logprob": -14.25,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0078125,
+          "logprob": -12.0234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7578125,
+          "logprob": -11.7734375,
           "text": "<image>"
         },
         {
@@ -37705,52 +22911,52 @@
         },
         {
           "id": 32000,
-          "logprob": -11.625,
+          "logprob": -11.609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.53125,
+          "logprob": -13.5234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.046875,
+          "logprob": -12.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6796875,
+          "logprob": -12.6953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7109375,
+          "logprob": -11.703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.890625,
+          "logprob": -11.8828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8359375,
+          "logprob": -11.84375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7734375,
+          "logprob": -11.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3671875,
+          "logprob": -12.3515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.3125,
+          "logprob": -14.3046875,
           "text": "<image>"
         },
         {
@@ -37760,22 +22966,22 @@
         },
         {
           "id": 32000,
-          "logprob": -11.921875,
+          "logprob": -11.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3203125,
+          "logprob": -11.328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1640625,
+          "logprob": -14.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.2421875,
+          "logprob": -11.25,
           "text": "<image>"
         },
         {
@@ -37785,7 +22991,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.078125,
           "text": "<image>"
         },
         {
@@ -37795,12 +23001,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.2578125,
+          "logprob": -12.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.7265625,
+          "logprob": -13.71875,
           "text": "<image>"
         },
         {
@@ -37820,12 +23026,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.3828125,
+          "logprob": -13.390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5390625,
+          "logprob": -12.515625,
           "text": "<image>"
         },
         {
@@ -37835,22 +23041,22 @@
         },
         {
           "id": 32000,
-          "logprob": -14.15625,
+          "logprob": -14.1796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5703125,
+          "logprob": -11.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5546875,
+          "logprob": -11.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.65625,
+          "logprob": -11.671875,
           "text": "<image>"
         },
         {
@@ -37860,47 +23066,47 @@
         },
         {
           "id": 32000,
-          "logprob": -12.6484375,
+          "logprob": -12.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5390625,
+          "logprob": -12.515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7734375,
+          "logprob": -11.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0859375,
+          "logprob": -13.1328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9140625,
+          "logprob": -12.8984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.6015625,
+          "logprob": -14.609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5390625,
+          "logprob": -11.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8828125,
+          "logprob": -11.8515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.078125,
+          "logprob": -13.0625,
           "text": "<image>"
         },
         {
@@ -37910,17 +23116,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.15625,
+          "logprob": -12.1328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.03125,
+          "logprob": -12.0234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4453125,
+          "logprob": -12.453125,
           "text": "<image>"
         },
         {
@@ -37930,7 +23136,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.9375,
+          "logprob": -11.9296875,
           "text": "<image>"
         },
         {
@@ -37945,42 +23151,42 @@
         },
         {
           "id": 32000,
-          "logprob": -12.7890625,
+          "logprob": -12.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0390625,
+          "logprob": -13.0234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8203125,
+          "logprob": -12.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.578125,
+          "logprob": -12.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.7578125,
+          "logprob": -10.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9609375,
+          "logprob": -11.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.484375,
+          "logprob": -11.4921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.265625,
+          "logprob": -11.25,
           "text": "<image>"
         },
         {
@@ -37990,17 +23196,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.71875,
+          "logprob": -12.6953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7890625,
+          "logprob": -11.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5234375,
+          "logprob": -11.515625,
           "text": "<image>"
         },
         {
@@ -38010,7 +23216,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.46875,
+          "logprob": -12.4765625,
           "text": "<image>"
         },
         {
@@ -38025,17 +23231,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.65625,
+          "logprob": -11.6484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5234375,
+          "logprob": -11.515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7890625,
+          "logprob": -12.8125,
           "text": "<image>"
         },
         {
@@ -38045,17 +23251,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.28125,
+          "logprob": -13.2890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.109375,
+          "logprob": -11.09375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.34375,
+          "logprob": -12.3359375,
           "text": "<image>"
         },
         {
@@ -38065,27 +23271,27 @@
         },
         {
           "id": 32000,
-          "logprob": -13.0,
+          "logprob": -13.0078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.28125,
+          "logprob": -15.328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.8046875,
+          "logprob": -14.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0234375,
+          "logprob": -11.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.109375,
+          "logprob": -13.0703125,
           "text": "<image>"
         },
         {
@@ -38095,12 +23301,12 @@
         },
         {
           "id": 32000,
-          "logprob": -14.0078125,
+          "logprob": -14.0859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.7109375,
+          "logprob": -14.734375,
           "text": "<image>"
         },
         {
@@ -38115,127 +23321,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.9765625,
+          "logprob": -13.953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.3046875,
+          "logprob": -14.28125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.78125,
+          "logprob": -14.8828125,
           "text": "<image>"
         },
         {
@@ -38245,22 +23341,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.859375,
+          "logprob": -12.28125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.765625,
+          "logprob": -12.34375,
           "text": "<image>"
         },
         {
@@ -38275,117 +23361,87 @@
         },
         {
           "id": 32000,
-          "logprob": -11.71875,
+          "logprob": -12.4296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.25,
+          "logprob": -11.8984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.65625,
+          "logprob": -11.984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.84375,
+          "logprob": -11.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0390625,
+          "logprob": -11.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4921875,
+          "logprob": -12.1875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8828125,
+          "logprob": -11.6171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1015625,
+          "logprob": -12.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.0625,
+          "logprob": -12.4765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.09375,
+          "logprob": -12.953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2578125,
+          "logprob": -10.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.4453125,
+          "logprob": -13.1015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9921875,
+          "logprob": -12.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.140625,
+          "logprob": -11.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.828125,
+          "logprob": -11.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5546875,
+          "logprob": -11.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6015625,
+          "logprob": -11.765625,
           "text": "<image>"
         },
         {
@@ -38395,32 +23451,182 @@
         },
         {
           "id": 32000,
-          "logprob": -12.84375,
+          "logprob": -12.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.78125,
+          "logprob": -11.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1171875,
+          "logprob": -12.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8046875,
+          "logprob": -11.71875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5234375,
+          "logprob": -11.34375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4609375,
+          "logprob": -12.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4765625,
           "text": "<image>"
         },
         {
@@ -38440,22 +23646,22 @@
         },
         {
           "id": 32000,
-          "logprob": -12.953125,
+          "logprob": -12.9609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.859375,
+          "logprob": -12.8515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0390625,
+          "logprob": -12.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.28125,
+          "logprob": -13.234375,
           "text": "<image>"
         },
         {
@@ -38465,12 +23671,12 @@
         },
         {
           "id": 32000,
-          "logprob": -10.8203125,
+          "logprob": -10.8515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.6953125,
+          "logprob": -10.671875,
           "text": "<image>"
         },
         {
@@ -38480,7 +23686,7 @@
         },
         {
           "id": 32000,
-          "logprob": -10.9375,
+          "logprob": -10.9140625,
           "text": "<image>"
         },
         {
@@ -38500,17 +23706,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.3046875,
+          "logprob": -12.2890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9609375,
+          "logprob": -10.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6328125,
+          "logprob": -11.6015625,
           "text": "<image>"
         },
         {
@@ -38520,17 +23726,17 @@
         },
         {
           "id": 32000,
-          "logprob": -10.84375,
+          "logprob": -10.8515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.140625,
+          "logprob": -11.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.09375,
+          "logprob": -11.0859375,
           "text": "<image>"
         },
         {
@@ -38540,12 +23746,12 @@
         },
         {
           "id": 32000,
-          "logprob": -15.578125,
+          "logprob": -15.765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.875,
+          "logprob": -11.8515625,
           "text": "<image>"
         },
         {
@@ -38560,22 +23766,22 @@
         },
         {
           "id": 32000,
-          "logprob": -10.984375,
+          "logprob": -10.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.234375,
+          "logprob": -12.25,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8359375,
+          "logprob": -11.828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8515625,
+          "logprob": -11.84375,
           "text": "<image>"
         },
         {
@@ -38590,7 +23796,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.5625,
+          "logprob": -13.5703125,
           "text": "<image>"
         },
         {
@@ -38600,37 +23806,37 @@
         },
         {
           "id": 32000,
-          "logprob": -13.5859375,
+          "logprob": -13.6171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.0234375,
+          "logprob": -14.0078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1796875,
+          "logprob": -11.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1484375,
+          "logprob": -14.140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.234375,
+          "logprob": -12.25,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.734375,
+          "logprob": -11.7265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3515625,
+          "logprob": -12.3203125,
           "text": "<image>"
         },
         {
@@ -38640,17 +23846,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.90625,
+          "logprob": -11.8828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3359375,
+          "logprob": -12.3125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5703125,
+          "logprob": -11.5625,
           "text": "<image>"
         },
         {
@@ -38660,17 +23866,17 @@
         },
         {
           "id": 32000,
-          "logprob": -14.6875,
+          "logprob": -14.6796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0625,
+          "logprob": -11.046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5234375,
+          "logprob": -11.5,
           "text": "<image>"
         },
         {
@@ -38695,12 +23901,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.46875,
+          "logprob": -12.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.0859375,
           "text": "<image>"
         },
         {
@@ -38713,21 +23919,6 @@
           "logprob": -12.46875,
           "text": "<image>"
         },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6640625,
-          "text": "<image>"
-        },
         {
           "id": 32000,
           "logprob": -11.9765625,
@@ -38735,7 +23926,22 @@
         },
         {
           "id": 32000,
-          "logprob": -10.8671875,
+          "logprob": -11.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.875,
           "text": "<image>"
         },
         {
@@ -38755,12 +23961,12 @@
         },
         {
           "id": 32000,
-          "logprob": -10.796875,
+          "logprob": -10.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.59375,
+          "logprob": -11.6171875,
           "text": "<image>"
         },
         {
@@ -38790,22 +23996,22 @@
         },
         {
           "id": 32000,
-          "logprob": -11.390625,
+          "logprob": -11.3828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9375,
+          "logprob": -10.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.6796875,
+          "logprob": -10.703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9375,
+          "logprob": -11.90625,
           "text": "<image>"
         },
         {
@@ -38815,82 +24021,57 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8125,
+          "logprob": -11.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3671875,
+          "logprob": -11.4453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -9.625,
+          "logprob": -9.6328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.8515625,
+          "logprob": -10.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9453125,
+          "logprob": -13.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8203125,
+          "logprob": -12.890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3359375,
+          "logprob": -13.4765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.015625,
+          "logprob": -14.109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8671875,
+          "logprob": -11.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4453125,
+          "logprob": -13.484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.75,
+          "logprob": -13.5625,
           "text": "<image>"
         },
         {
@@ -38898,6 +24079,31 @@
           "logprob": -12.859375,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -10.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.890625,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -13.0625,
@@ -38905,7 +24111,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.21875,
+          "logprob": -11.2265625,
           "text": "<image>"
         },
         {
@@ -38915,7 +24121,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.296875,
+          "logprob": -12.2890625,
           "text": "<image>"
         },
         {
@@ -38925,12 +24131,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.8671875,
+          "logprob": -12.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.6953125,
+          "logprob": -15.671875,
           "text": "<image>"
         },
         {
@@ -38945,7 +24151,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.140625,
+          "logprob": -12.1328125,
           "text": "<image>"
         },
         {
@@ -38955,22 +24161,22 @@
         },
         {
           "id": 32000,
-          "logprob": -11.390625,
+          "logprob": -11.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.328125,
+          "logprob": -12.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.265625,
+          "logprob": -12.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9453125,
+          "logprob": -10.9296875,
           "text": "<image>"
         },
         {
@@ -38985,7 +24191,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.6015625,
+          "logprob": -11.59375,
           "text": "<image>"
         },
         {
@@ -38995,7 +24201,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.546875,
+          "logprob": -11.53125,
           "text": "<image>"
         },
         {
@@ -39010,7 +24216,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.6796875,
+          "logprob": -11.6875,
           "text": "<image>"
         },
         {
@@ -39025,27 +24231,27 @@
         },
         {
           "id": 32000,
-          "logprob": -12.125,
+          "logprob": -12.1171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5703125,
+          "logprob": -12.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.109375,
+          "logprob": -12.1015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0078125,
+          "logprob": -12.0,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.734375,
+          "logprob": -13.7265625,
           "text": "<image>"
         },
         {
@@ -39055,67 +24261,67 @@
         },
         {
           "id": 32000,
-          "logprob": -11.6484375,
+          "logprob": -11.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.34375,
+          "logprob": -12.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.140625,
+          "logprob": -12.1484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1796875,
+          "logprob": -14.21875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.8046875,
+          "logprob": -14.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -18.046875,
+          "logprob": -17.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.34375,
+          "logprob": -15.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.4453125,
+          "logprob": -14.4765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.625,
+          "logprob": -10.6015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0546875,
+          "logprob": -12.0703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.859375,
+          "logprob": -12.84375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2265625,
+          "logprob": -12.2421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2109375,
+          "logprob": -13.203125,
           "text": "<image>"
         },
         {
@@ -39125,37 +24331,37 @@
         },
         {
           "id": 32000,
-          "logprob": -10.9296875,
+          "logprob": -10.9375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9140625,
+          "logprob": -11.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.125,
+          "logprob": -12.1015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9609375,
+          "logprob": -10.9453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7734375,
+          "logprob": -11.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.828125,
+          "logprob": -11.8203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2890625,
+          "logprob": -12.296875,
           "text": "<image>"
         },
         {
@@ -39165,27 +24371,27 @@
         },
         {
           "id": 32000,
-          "logprob": -11.53125,
+          "logprob": -11.5390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0234375,
+          "logprob": -12.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.2421875,
+          "logprob": -15.2265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1171875,
+          "logprob": -12.125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1171875,
+          "logprob": -12.125,
           "text": "<image>"
         },
         {
@@ -39195,7 +24401,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.671875,
+          "logprob": -11.6796875,
           "text": "<image>"
         },
         {
@@ -39215,32 +24421,32 @@
         },
         {
           "id": 32000,
-          "logprob": -12.734375,
+          "logprob": -12.703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.609375,
+          "logprob": -11.6328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5234375,
+          "logprob": -11.515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5078125,
+          "logprob": -11.546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2109375,
+          "logprob": -12.21875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.421875,
+          "logprob": -11.4140625,
           "text": "<image>"
         },
         {
@@ -39250,137 +24456,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.96875,
+          "logprob": -11.9765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.46875,
+          "logprob": -12.9609375,
           "text": "<image>"
         },
         {
@@ -39388,6 +24469,131 @@
           "logprob": -12.578125,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -11.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5078125,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -11.6171875,
@@ -39395,32 +24601,32 @@
         },
         {
           "id": 32000,
-          "logprob": -13.8125,
+          "logprob": -13.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1640625,
+          "logprob": -11.203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.765625,
+          "logprob": -11.78125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.5,
+          "logprob": -16.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3046875,
+          "logprob": -12.296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1171875,
+          "logprob": -14.1484375,
           "text": "<image>"
         },
         {
@@ -39430,12 +24636,47 @@
         },
         {
           "id": 32000,
-          "logprob": -12.5,
+          "logprob": -12.4921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.6015625,
+          "logprob": -10.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1953125,
           "text": "<image>"
         },
         {
@@ -39445,52 +24686,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.625,
+          "logprob": -11.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3046875,
+          "logprob": -12.6484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1171875,
+          "logprob": -11.109375,
           "text": "<image>"
         },
         {
@@ -39500,7 +24706,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.921875,
+          "logprob": -13.90625,
           "text": "<image>"
         },
         {
@@ -39515,12 +24721,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.3671875,
+          "logprob": -13.375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.7109375,
+          "logprob": -14.703125,
           "text": "<image>"
         },
         {
@@ -39530,7 +24736,7 @@
         },
         {
           "id": 32000,
-          "logprob": -14.859375,
+          "logprob": -14.890625,
           "text": "<image>"
         },
         {
@@ -39545,37 +24751,37 @@
         },
         {
           "id": 32000,
-          "logprob": -11.9296875,
+          "logprob": -11.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3828125,
+          "logprob": -13.3984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.046875,
+          "logprob": -13.9453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1484375,
+          "logprob": -13.1875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.265625,
+          "logprob": -11.28125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.125,
+          "logprob": -15.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.046875,
+          "logprob": -15.0234375,
           "text": "<image>"
         },
         {
@@ -39590,92 +24796,22 @@
         },
         {
           "id": 32000,
-          "logprob": -14.09375,
+          "logprob": -14.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.890625,
+          "logprob": -13.8984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.3203125,
+          "logprob": -15.3125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9609375,
+          "logprob": -12.8125,
           "text": "<image>"
         },
         {
@@ -39685,22 +24821,92 @@
         },
         {
           "id": 32000,
-          "logprob": -12.3203125,
+          "logprob": -13.046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.59375,
+          "logprob": -10.625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.171875,
+          "logprob": -11.25,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5,
+          "logprob": -12.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4921875,
           "text": "<image>"
         },
         {
@@ -39730,17 +24936,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.78125,
+          "logprob": -12.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3125,
+          "logprob": -11.3046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1171875,
+          "logprob": -11.109375,
           "text": "<image>"
         },
         {
@@ -39750,27 +24956,27 @@
         },
         {
           "id": 32000,
-          "logprob": -13.0859375,
+          "logprob": -13.0234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.921875,
+          "logprob": -10.9296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.515625,
+          "logprob": -13.5078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4921875,
+          "logprob": -11.5,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8203125,
+          "logprob": -11.8359375,
           "text": "<image>"
         },
         {
@@ -39780,42 +24986,42 @@
         },
         {
           "id": 32000,
-          "logprob": -14.0078125,
+          "logprob": -14.0234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.21875,
+          "logprob": -15.2578125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -17.84375,
+          "logprob": -17.953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.046875,
+          "logprob": -12.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.296875,
+          "logprob": -12.28125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7109375,
+          "logprob": -11.6875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1015625,
+          "logprob": -12.1171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.515625,
+          "logprob": -11.5546875,
           "text": "<image>"
         },
         {
@@ -39825,17 +25031,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.640625,
+          "logprob": -12.6171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4609375,
+          "logprob": -13.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.84375,
+          "logprob": -14.828125,
           "text": "<image>"
         },
         {
@@ -39845,22 +25051,22 @@
         },
         {
           "id": 32000,
-          "logprob": -11.6171875,
+          "logprob": -11.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.765625,
+          "logprob": -12.7421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5390625,
+          "logprob": -13.484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4453125,
+          "logprob": -11.4609375,
           "text": "<image>"
         },
         {
@@ -39870,37 +25076,37 @@
         },
         {
           "id": 32000,
-          "logprob": -11.96875,
+          "logprob": -11.984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4765625,
+          "logprob": -12.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.890625,
+          "logprob": -15.9375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.953125,
+          "logprob": -11.984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.40625,
+          "logprob": -11.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0,
+          "logprob": -12.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1875,
+          "logprob": -14.3046875,
           "text": "<image>"
         },
         {
@@ -39910,7 +25116,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.125,
+          "logprob": -12.1171875,
           "text": "<image>"
         },
         {
@@ -39920,22 +25126,22 @@
         },
         {
           "id": 32000,
-          "logprob": -13.140625,
+          "logprob": -13.1484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1484375,
+          "logprob": -12.1875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.375,
+          "logprob": -13.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.9453125,
+          "logprob": -13.9921875,
           "text": "<image>"
         },
         {
@@ -39945,12 +25151,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0859375,
+          "logprob": -12.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.734375,
+          "logprob": -10.7421875,
           "text": "<image>"
         },
         {
@@ -39960,12 +25166,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.65625,
+          "logprob": -12.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.953125,
+          "logprob": -11.9765625,
           "text": "<image>"
         },
         {
@@ -39975,12 +25181,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.5,
+          "logprob": -12.4765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8984375,
+          "logprob": -11.890625,
           "text": "<image>"
         },
         {
@@ -40000,22 +25206,22 @@
         },
         {
           "id": 32000,
-          "logprob": -12.4375,
+          "logprob": -12.4609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5703125,
+          "logprob": -12.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.828125,
+          "logprob": -12.8203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.375,
+          "logprob": -13.359375,
           "text": "<image>"
         },
         {
@@ -40035,7 +25241,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.2421875,
+          "logprob": -11.234375,
           "text": "<image>"
         },
         {
@@ -40045,27 +25251,27 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.328125,
+          "logprob": -15.3046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.890625,
+          "logprob": -12.8828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4453125,
+          "logprob": -13.453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8203125,
+          "logprob": -11.8125,
           "text": "<image>"
         },
         {
@@ -40075,17 +25281,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.96875,
+          "logprob": -12.9765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9921875,
+          "logprob": -11.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.421875,
+          "logprob": -13.4296875,
           "text": "<image>"
         },
         {
@@ -40095,12 +25301,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.265625,
+          "logprob": -13.28125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.265625,
+          "logprob": -13.2578125,
           "text": "<image>"
         },
         {
@@ -40115,12 +25321,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.15625,
+          "logprob": -12.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.375,
+          "logprob": -10.3828125,
           "text": "<image>"
         },
         {
@@ -40130,7 +25336,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.2109375,
+          "logprob": -13.203125,
           "text": "<image>"
         },
         {
@@ -40140,7 +25346,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.15625,
+          "logprob": -12.1328125,
           "text": "<image>"
         },
         {
@@ -40150,12 +25356,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.171875,
+          "logprob": -13.1796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6328125,
+          "logprob": -11.65625,
           "text": "<image>"
         },
         {
@@ -40165,12 +25371,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.90625,
+          "logprob": -11.9375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.765625,
+          "logprob": -11.7734375,
           "text": "<image>"
         },
         {
@@ -40185,7 +25391,7 @@
         },
         {
           "id": 32000,
-          "logprob": -10.8984375,
+          "logprob": -10.90625,
           "text": "<image>"
         },
         {
@@ -40195,17 +25401,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.40625,
+          "logprob": -12.3828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5390625,
+          "logprob": -11.5234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.59375,
+          "logprob": -16.578125,
           "text": "<image>"
         },
         {
@@ -40215,7 +25421,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.9140625,
+          "logprob": -11.9453125,
           "text": "<image>"
         },
         {
@@ -40230,22 +25436,22 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8984375,
+          "logprob": -11.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0703125,
+          "logprob": -13.0625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.03125,
+          "logprob": -12.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.921875,
+          "logprob": -11.9296875,
           "text": "<image>"
         },
         {
@@ -40255,12 +25461,12 @@
         },
         {
           "id": 32000,
-          "logprob": -10.5390625,
+          "logprob": -10.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.453125,
+          "logprob": -11.4375,
           "text": "<image>"
         },
         {
@@ -40270,22 +25476,22 @@
         },
         {
           "id": 32000,
-          "logprob": -13.0859375,
+          "logprob": -13.09375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1796875,
+          "logprob": -12.15625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.8203125,
+          "logprob": -15.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.984375,
+          "logprob": -10.9765625,
           "text": "<image>"
         },
         {
@@ -40295,42 +25501,42 @@
         },
         {
           "id": 32000,
-          "logprob": -10.109375,
+          "logprob": -10.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8671875,
+          "logprob": -11.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1328125,
+          "logprob": -13.125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.640625,
+          "logprob": -12.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.015625,
+          "logprob": -16.046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5078125,
+          "logprob": -11.515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.7265625,
+          "logprob": -13.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.703125,
+          "logprob": -14.671875,
           "text": "<image>"
         },
         {
@@ -40340,32 +25546,32 @@
         },
         {
           "id": 32000,
-          "logprob": -12.2421875,
+          "logprob": -12.2109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.6640625,
+          "logprob": -10.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.6640625,
+          "logprob": -15.6484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9140625,
+          "logprob": -10.9296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3203125,
+          "logprob": -11.328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.046875,
+          "logprob": -11.0546875,
           "text": "<image>"
         },
         {
@@ -40375,7 +25581,7 @@
         },
         {
           "id": 32000,
-          "logprob": -10.953125,
+          "logprob": -10.96875,
           "text": "<image>"
         },
         {
@@ -40390,337 +25596,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2578125,
+          "logprob": -12.359375,
           "text": "<image>"
         },
         {
@@ -40730,22 +25606,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.84375,
+          "logprob": -14.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
+          "logprob": -11.4921875,
           "text": "<image>"
         },
         {
@@ -40755,87 +25621,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.125,
+          "logprob": -12.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.390625,
+          "logprob": -12.640625,
           "text": "<image>"
         },
         {
@@ -40845,87 +25636,297 @@
         },
         {
           "id": 32000,
-          "logprob": -12.5078125,
+          "logprob": -12.4375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.09375,
+          "logprob": -11.8828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.8515625,
+          "logprob": -12.15625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8203125,
+          "logprob": -11.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.734375,
+          "logprob": -13.0,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2578125,
+          "logprob": -12.0703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.28125,
+          "logprob": -11.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.859375,
+          "logprob": -13.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.703125,
+          "logprob": -12.1484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5703125,
+          "logprob": -13.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.59375,
+          "logprob": -12.203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.890625,
+          "logprob": -11.9375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3359375,
+          "logprob": -12.1796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0625,
+          "logprob": -10.7421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.53125,
+          "logprob": -10.8203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7109375,
+          "logprob": -12.375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.7578125,
+          "logprob": -11.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.25,
           "text": "<image>"
         },
         {
@@ -40935,17 +25936,47 @@
         },
         {
           "id": 32000,
-          "logprob": -10.75,
+          "logprob": -11.8515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.2265625,
+          "logprob": -13.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9296875,
+          "logprob": -10.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1328125,
           "text": "<image>"
         },
         {
@@ -40955,7 +25986,12 @@
         },
         {
           "id": 32000,
-          "logprob": -10.5390625,
+          "logprob": -14.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0546875,
           "text": "<image>"
         },
         {
@@ -40963,6 +25999,56 @@
           "logprob": -11.6796875,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -11.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.234375,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -12.5390625,
@@ -40970,7 +26056,127 @@
         },
         {
           "id": 32000,
-          "logprob": -16.953125,
+          "logprob": -15.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.0,
           "text": "<image>"
         },
         {
@@ -40980,12 +26186,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.3359375,
+          "logprob": -11.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.671875,
+          "logprob": -11.6796875,
           "text": "<image>"
         },
         {
@@ -41000,12 +26206,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8671875,
+          "logprob": -11.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.484375,
+          "logprob": -11.4765625,
           "text": "<image>"
         },
         {
@@ -41015,12 +26221,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.6640625,
+          "logprob": -12.6875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4296875,
+          "logprob": -13.421875,
           "text": "<image>"
         },
         {
@@ -41035,7 +26241,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.3046875,
+          "logprob": -12.3125,
           "text": "<image>"
         },
         {
@@ -41045,17 +26251,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.34375,
+          "logprob": -13.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0703125,
+          "logprob": -13.0625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.234375,
+          "logprob": -11.2421875,
           "text": "<image>"
         },
         {
@@ -41065,37 +26271,37 @@
         },
         {
           "id": 32000,
-          "logprob": -13.4921875,
+          "logprob": -13.5,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.7578125,
+          "logprob": -10.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.53125,
+          "logprob": -13.5390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6953125,
+          "logprob": -12.703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.203125,
+          "logprob": -13.1953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -17.078125,
+          "logprob": -17.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.25,
+          "logprob": -15.0234375,
           "text": "<image>"
         },
         {
@@ -41105,17 +26311,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.140625,
+          "logprob": -12.125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5234375,
+          "logprob": -11.515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.0625,
+          "logprob": -16.125,
           "text": "<image>"
         },
         {
@@ -41125,17 +26331,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.6015625,
+          "logprob": -11.609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.5859375,
+          "logprob": -14.6015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.890625,
+          "logprob": -11.8984375,
           "text": "<image>"
         },
         {
@@ -41150,22 +26356,22 @@
         },
         {
           "id": 32000,
-          "logprob": -11.7734375,
+          "logprob": -11.78125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9921875,
+          "logprob": -13.0,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9375,
+          "logprob": -12.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9921875,
+          "logprob": -12.0,
           "text": "<image>"
         },
         {
@@ -41175,7 +26381,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.7578125,
+          "logprob": -11.7734375,
           "text": "<image>"
         },
         {
@@ -41185,7 +26391,7 @@
         },
         {
           "id": 32000,
-          "logprob": -14.328125,
+          "logprob": -14.3203125,
           "text": "<image>"
         },
         {
@@ -41195,22 +26401,22 @@
         },
         {
           "id": 32000,
-          "logprob": -13.84375,
+          "logprob": -13.875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.2109375,
+          "logprob": -14.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.390625,
+          "logprob": -12.375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.953125,
+          "logprob": -13.9375,
           "text": "<image>"
         },
         {
@@ -41220,27 +26426,27 @@
         },
         {
           "id": 32000,
-          "logprob": -14.6796875,
+          "logprob": -14.609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.84375,
+          "logprob": -11.8359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.65625,
+          "logprob": -10.6484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0078125,
+          "logprob": -12.0,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0859375,
+          "logprob": -11.0703125,
           "text": "<image>"
         },
         {
@@ -41260,7 +26466,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.6875,
+          "logprob": -12.6640625,
           "text": "<image>"
         },
         {
@@ -41275,22 +26481,22 @@
         },
         {
           "id": 32000,
-          "logprob": -16.109375,
+          "logprob": -16.09375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.984375,
+          "logprob": -11.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.40625,
+          "logprob": -13.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.375,
+          "logprob": -13.453125,
           "text": "<image>"
         },
         {
@@ -41300,57 +26506,57 @@
         },
         {
           "id": 32000,
-          "logprob": -12.921875,
+          "logprob": -12.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.859375,
+          "logprob": -14.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3671875,
+          "logprob": -13.375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0546875,
+          "logprob": -12.0703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.21875,
+          "logprob": -15.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6953125,
+          "logprob": -11.6875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.5234375,
+          "logprob": -10.5390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.8984375,
+          "logprob": -14.703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.8515625,
+          "logprob": -13.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6796875,
+          "logprob": -11.6875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -17.4375,
+          "logprob": -17.453125,
           "text": "<image>"
         },
         {
@@ -41365,7 +26571,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8125,
+          "logprob": -11.7890625,
           "text": "<image>"
         },
         {
@@ -41380,12 +26586,12 @@
         },
         {
           "id": 32000,
-          "logprob": -14.2421875,
+          "logprob": -14.265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.7265625,
+          "logprob": -13.6953125,
           "text": "<image>"
         },
         {
@@ -41400,57 +26606,57 @@
         },
         {
           "id": 32000,
-          "logprob": -12.84375,
+          "logprob": -12.890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9140625,
+          "logprob": -11.9375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.7421875,
+          "logprob": -14.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.984375,
+          "logprob": -11.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.71875,
+          "logprob": -14.7265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.578125,
+          "logprob": -11.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.015625,
+          "logprob": -13.0,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.796875,
+          "logprob": -12.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0078125,
+          "logprob": -11.015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.46875,
+          "logprob": -16.421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9296875,
+          "logprob": -11.921875,
           "text": "<image>"
         },
         {
@@ -41460,7 +26666,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.8515625,
+          "logprob": -12.859375,
           "text": "<image>"
         },
         {
@@ -41475,12 +26681,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.140625,
+          "logprob": -12.1328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.359375,
+          "logprob": -13.3515625,
           "text": "<image>"
         },
         {
@@ -41495,27 +26701,27 @@
         },
         {
           "id": 32000,
-          "logprob": -11.578125,
+          "logprob": -11.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8203125,
+          "logprob": -12.84375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.984375,
+          "logprob": -11.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5078125,
+          "logprob": -12.515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.625,
+          "logprob": -12.6328125,
           "text": "<image>"
         },
         {
@@ -41530,7 +26736,7 @@
         },
         {
           "id": 32000,
-          "logprob": -14.0546875,
+          "logprob": -14.046875,
           "text": "<image>"
         },
         {
@@ -41540,7 +26746,7 @@
         },
         {
           "id": 32000,
-          "logprob": -17.21875,
+          "logprob": -17.203125,
           "text": "<image>"
         },
         {
@@ -41550,17 +26756,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.3671875,
+          "logprob": -13.375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.6171875,
+          "logprob": -14.609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9609375,
+          "logprob": -12.953125,
           "text": "<image>"
         },
         {
@@ -41570,12 +26776,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.7265625,
+          "logprob": -11.71875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.03125,
+          "logprob": -15.0234375,
           "text": "<image>"
         },
         {
@@ -41585,7 +26791,7 @@
         },
         {
           "id": 32000,
-          "logprob": -15.890625,
+          "logprob": -15.9296875,
           "text": "<image>"
         },
         {
@@ -41610,22 +26816,22 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0234375,
+          "logprob": -12.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7578125,
+          "logprob": -12.75,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.7578125,
+          "logprob": -14.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5,
+          "logprob": -11.484375,
           "text": "<image>"
         },
         {
@@ -41635,7 +26841,7 @@
         },
         {
           "id": 32000,
-          "logprob": -10.7265625,
+          "logprob": -10.734375,
           "text": "<image>"
         },
         {
@@ -41645,22 +26851,22 @@
         },
         {
           "id": 32000,
-          "logprob": -11.4140625,
+          "logprob": -11.421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9453125,
+          "logprob": -12.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1171875,
+          "logprob": -13.0625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0390625,
+          "logprob": -12.03125,
           "text": "<image>"
         },
         {
@@ -41670,7 +26876,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.234375,
+          "logprob": -12.2109375,
           "text": "<image>"
         },
         {
@@ -41685,7 +26891,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.0390625,
+          "logprob": -13.09375,
           "text": "<image>"
         },
         {
@@ -41695,7 +26901,7 @@
         },
         {
           "id": 32000,
-          "logprob": -15.265625,
+          "logprob": -15.2578125,
           "text": "<image>"
         },
         {
@@ -41710,7 +26916,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.453125,
+          "logprob": -11.4609375,
           "text": "<image>"
         },
         {
@@ -41725,12 +26931,12 @@
         },
         {
           "id": 32000,
-          "logprob": -10.6796875,
+          "logprob": -10.6875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.0546875,
           "text": "<image>"
         },
         {
@@ -41740,7 +26946,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.421875,
+          "logprob": -12.4140625,
           "text": "<image>"
         },
         {
@@ -41750,27 +26956,27 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8046875,
+          "logprob": -11.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.3828125,
+          "logprob": -14.4609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.734375,
+          "logprob": -11.7109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.890625,
+          "logprob": -12.828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.859375,
+          "logprob": -12.8359375,
           "text": "<image>"
         },
         {
@@ -41780,27 +26986,27 @@
         },
         {
           "id": 32000,
-          "logprob": -13.5625,
+          "logprob": -13.5390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.25,
+          "logprob": -16.21875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.8125,
+          "logprob": -16.84375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.921875,
+          "logprob": -11.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.375,
+          "logprob": -14.359375,
           "text": "<image>"
         },
         {
@@ -41810,12 +27016,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.671875,
+          "logprob": -13.6953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9921875,
+          "logprob": -12.984375,
           "text": "<image>"
         },
         {
@@ -41830,57 +27036,37 @@
         },
         {
           "id": 32000,
-          "logprob": -17.5625,
+          "logprob": -17.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.625,
+          "logprob": -12.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.0625,
+          "logprob": -15.046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.75,
+          "logprob": -16.71875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.84375,
+          "logprob": -11.8359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.328125,
+          "logprob": -13.3515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
+          "logprob": -12.2734375,
           "text": "<image>"
         },
         {
@@ -41890,17 +27076,37 @@
         },
         {
           "id": 32000,
-          "logprob": -13.109375,
+          "logprob": -11.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.625,
+          "logprob": -12.46875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.890625,
+          "logprob": -11.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.921875,
           "text": "<image>"
         },
         {
@@ -41910,7 +27116,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.15625,
+          "logprob": -13.21875,
           "text": "<image>"
         },
         {
@@ -41930,7 +27136,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.5390625,
+          "logprob": -12.5625,
           "text": "<image>"
         },
         {
@@ -41940,12 +27146,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.9609375,
+          "logprob": -11.9453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.828125,
+          "logprob": -11.8359375,
           "text": "<image>"
         },
         {
@@ -41955,22 +27161,22 @@
         },
         {
           "id": 32000,
-          "logprob": -12.8125,
+          "logprob": -12.875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.96875,
+          "logprob": -11.9609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.109375,
+          "logprob": -13.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6328125,
+          "logprob": -12.625,
           "text": "<image>"
         },
         {
@@ -41990,12 +27196,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.109375,
+          "logprob": -12.0859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.265625,
+          "logprob": -14.25,
           "text": "<image>"
         },
         {
@@ -42005,127 +27211,17 @@
         },
         {
           "id": 32000,
-          "logprob": -16.640625,
+          "logprob": -16.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.828125,
+          "logprob": -12.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -19.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5859375,
+          "logprob": -12.7890625,
           "text": "<image>"
         },
         {
@@ -42135,852 +27231,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.3046875,
+          "logprob": -12.3125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5234375,
+          "logprob": -12.3203125,
           "text": "<image>"
         },
         {
@@ -42990,107 +27246,137 @@
         },
         {
           "id": 32000,
-          "logprob": -11.6875,
+          "logprob": -14.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.765625,
+          "logprob": -14.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4140625,
+          "logprob": -12.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -18.703125,
+          "logprob": -14.546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.6484375,
+          "logprob": -19.4375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.2109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.703125,
+          "logprob": -13.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.0859375,
+          "logprob": -11.8515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.0703125,
+          "logprob": -11.4453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.5234375,
+          "logprob": -12.2265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3671875,
+          "logprob": -12.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0390625,
+          "logprob": -12.3359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.640625,
+          "logprob": -17.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8359375,
+          "logprob": -17.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1015625,
+          "logprob": -11.515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3515625,
+          "logprob": -12.0078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.578125,
+          "logprob": -13.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.25,
+          "logprob": -12.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.15625,
+          "logprob": -12.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7265625,
+          "logprob": -12.328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.90625,
+          "logprob": -13.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.5,
           "text": "<image>"
         },
         {
@@ -43105,17 +27391,32 @@
         },
         {
           "id": 32000,
-          "logprob": -10.5625,
+          "logprob": -14.875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.171875,
+          "logprob": -11.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2265625,
+          "logprob": -11.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8984375,
           "text": "<image>"
         },
         {
@@ -43125,137 +27426,37 @@
         },
         {
           "id": 32000,
-          "logprob": -11.7578125,
+          "logprob": -11.328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.875,
+          "logprob": -11.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4453125,
+          "logprob": -11.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1796875,
+          "logprob": -15.046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9453125,
+          "logprob": -12.2109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8203125,
+          "logprob": -12.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4765625,
+          "logprob": -16.875,
           "text": "<image>"
         },
         {
@@ -43265,22 +27466,227 @@
         },
         {
           "id": 32000,
-          "logprob": -15.484375,
+          "logprob": -14.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.6328125,
+          "logprob": -11.9609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.6015625,
+          "logprob": -12.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.75,
+          "logprob": -11.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.890625,
           "text": "<image>"
         },
         {
@@ -43290,7 +27696,807 @@
         },
         {
           "id": 32000,
-          "logprob": -13.3125,
+          "logprob": -13.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -18.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.0859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2578125,
           "text": "<image>"
         },
         {
@@ -43300,12 +28506,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.484375,
+          "logprob": -13.5390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2734375,
+          "logprob": -13.265625,
           "text": "<image>"
         },
         {
@@ -43320,17 +28526,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.6796875,
+          "logprob": -11.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.53125,
+          "logprob": -16.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1484375,
+          "logprob": -14.171875,
           "text": "<image>"
         },
         {
@@ -43340,17 +28546,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8125,
+          "logprob": -11.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.03125,
+          "logprob": -15.1171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.484375,
+          "logprob": -15.4453125,
           "text": "<image>"
         },
         {
@@ -43365,37 +28571,37 @@
         },
         {
           "id": 32000,
-          "logprob": -10.6953125,
+          "logprob": -10.703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1640625,
+          "logprob": -11.15625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.84375,
+          "logprob": -12.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5625,
+          "logprob": -12.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7421875,
+          "logprob": -12.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -18.0,
+          "logprob": -17.953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5859375,
+          "logprob": -11.59375,
           "text": "<image>"
         },
         {
@@ -43410,12 +28616,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.6796875,
+          "logprob": -11.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.71875,
+          "logprob": -12.8515625,
           "text": "<image>"
         },
         {
@@ -43425,27 +28631,27 @@
         },
         {
           "id": 32000,
-          "logprob": -12.1875,
+          "logprob": -12.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7578125,
+          "logprob": -12.7265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.125,
+          "logprob": -15.140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.9140625,
+          "logprob": -14.9375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.546875,
+          "logprob": -15.6875,
           "text": "<image>"
         },
         {
@@ -43455,22 +28661,22 @@
         },
         {
           "id": 32000,
-          "logprob": -14.109375,
+          "logprob": -14.125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.234375,
+          "logprob": -13.2265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.15625,
+          "logprob": -13.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.734375,
+          "logprob": -12.703125,
           "text": "<image>"
         },
         {
@@ -43480,7 +28686,7 @@
         },
         {
           "id": 32000,
-          "logprob": -14.7734375,
+          "logprob": -14.765625,
           "text": "<image>"
         },
         {
@@ -43490,12 +28696,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.7421875,
+          "logprob": -11.71875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.921875,
+          "logprob": -11.9296875,
           "text": "<image>"
         },
         {
@@ -43505,17 +28711,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.6015625,
+          "logprob": -13.6328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4375,
+          "logprob": -11.4296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5,
+          "logprob": -13.5078125,
           "text": "<image>"
         },
         {
@@ -43530,27 +28736,27 @@
         },
         {
           "id": 32000,
-          "logprob": -12.1328125,
+          "logprob": -12.15625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.90625,
+          "logprob": -13.1171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.578125,
+          "logprob": -12.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3359375,
+          "logprob": -13.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7265625,
+          "logprob": -11.734375,
           "text": "<image>"
         },
         {
@@ -43558,491 +28764,11 @@
           "logprob": -10.953125,
           "text": "<image>"
         },
-        {
-          "id": 32000,
-          "logprob": -12.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -19.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.796875,
-          "text": "<image>"
-        },
         {
           "id": 32000,
           "logprob": -12.75,
           "text": "<image>"
         },
-        {
-          "id": 32000,
-          "logprob": -14.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.1953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.515625,
-          "text": "<image>"
-        },
         {
           "id": 32000,
           "logprob": -12.7734375,
@@ -44050,7 +28776,22 @@
         },
         {
           "id": 32000,
-          "logprob": -12.4609375,
+          "logprob": -13.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5390625,
           "text": "<image>"
         },
         {
@@ -44060,87 +28801,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.171875,
+          "logprob": -19.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8828125,
+          "logprob": -12.4765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8828125,
+          "logprob": -11.609375,
           "text": "<image>"
         },
         {
@@ -44153,6 +28824,541 @@
           "logprob": -13.3203125,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -12.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.359375,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -13.2109375,
@@ -44160,7 +29366,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.9765625,
+          "logprob": -11.984375,
           "text": "<image>"
         },
         {
@@ -44170,12 +29376,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0078125,
+          "logprob": -12.015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0390625,
+          "logprob": -11.03125,
           "text": "<image>"
         },
         {
@@ -44190,17 +29396,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.8203125,
+          "logprob": -13.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5078125,
+          "logprob": -13.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.734375,
+          "logprob": -11.7265625,
           "text": "<image>"
         },
         {
@@ -44210,22 +29416,22 @@
         },
         {
           "id": 32000,
-          "logprob": -17.3125,
+          "logprob": -16.84375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5234375,
+          "logprob": -12.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -17.625,
+          "logprob": -17.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9296875,
+          "logprob": -11.9453125,
           "text": "<image>"
         },
         {
@@ -44235,17 +29441,17 @@
         },
         {
           "id": 32000,
-          "logprob": -15.9140625,
+          "logprob": -16.0,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.65625,
+          "logprob": -16.625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5,
+          "logprob": -12.4921875,
           "text": "<image>"
         },
         {
@@ -44255,17 +29461,17 @@
         },
         {
           "id": 368,
-          "logprob": -0.19726562,
+          "logprob": -0.19604492,
           "text": "you"
         },
         {
           "id": 1912,
-          "logprob": -1.4990234,
+          "logprob": -1.5058594,
           "text": "tell"
         },
         {
           "id": 528,
-          "logprob": -0.31152344,
+          "logprob": -0.31030273,
           "text": "me"
         },
         {
@@ -44280,22 +29486,22 @@
         },
         {
           "id": 2485,
-          "logprob": -0.9941406,
+          "logprob": -0.9975586,
           "text": "short"
         },
         {
           "id": 2838,
-          "logprob": -0.46118164,
+          "logprob": -0.4633789,
           "text": "story"
         },
         {
           "id": 2818,
-          "logprob": -3.3183594,
+          "logprob": -3.3144531,
           "text": "based"
         },
         {
           "id": 356,
-          "logprob": -0.029129028,
+          "logprob": -0.029037476,
           "text": "on"
         },
         {
@@ -44305,12 +29511,12 @@
         },
         {
           "id": 3469,
-          "logprob": -0.29052734,
+          "logprob": -0.2890625,
           "text": "image"
         },
         {
           "id": 28804,
-          "logprob": -0.43188477,
+          "logprob": -0.42895508,
           "text": "?"
         }
       ],
@@ -44318,61 +29524,61 @@
       "tokens": [
         {
           "id": 13,
-          "logprob": -0.0076828003,
+          "logprob": -0.007621765,
           "special": false,
           "text": "\n"
         },
         {
           "id": 13,
-          "logprob": -0.20092773,
+          "logprob": -0.20275879,
           "special": false,
           "text": "\n"
         },
         {
           "id": 16114,
-          "logprob": -1.2587891,
+          "logprob": -1.2578125,
           "special": false,
           "text": "Once"
         },
         {
           "id": 3714,
-          "logprob": -0.20861816,
+          "logprob": -0.2084961,
           "special": false,
           "text": " upon"
         },
         {
           "id": 264,
-          "logprob": -0.0017719269,
+          "logprob": -0.0017738342,
           "special": false,
           "text": " a"
         },
         {
           "id": 727,
-          "logprob": -0.011909485,
+          "logprob": -0.011932373,
           "special": false,
           "text": " time"
         },
         {
           "id": 28725,
-          "logprob": -0.17529297,
+          "logprob": -0.17297363,
           "special": false,
           "text": ","
         },
         {
           "id": 736,
-          "logprob": -0.9082031,
+          "logprob": -0.9057617,
           "special": false,
           "text": " there"
         },
         {
           "id": 403,
-          "logprob": -0.057525635,
+          "logprob": -0.05758667,
           "special": false,
           "text": " was"
         },
         {
           "id": 264,
-          "logprob": -0.009651184,
+          "logprob": -0.00970459,
           "special": false,
           "text": " a"
         }
@@ -44394,7 +29600,7 @@
         },
         {
           "id": 1247,
-          "logprob": -2.3886719,
+          "logprob": -2.390625,
           "text": "User"
         },
         {
@@ -44409,12 +29615,12 @@
         },
         {
           "id": 32000,
-          "logprob": -10.671875,
+          "logprob": -10.6953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.7109375,
+          "logprob": -15.828125,
           "text": "<image>"
         },
         {
@@ -44424,82 +29630,82 @@
         },
         {
           "id": 32000,
-          "logprob": -10.0234375,
+          "logprob": -10.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.1328125,
+          "logprob": -10.140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.421875,
+          "logprob": -10.3125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.90625,
+          "logprob": -12.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.59375,
+          "logprob": -15.4921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.828125,
+          "logprob": -13.8671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.390625,
+          "logprob": -11.3671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.1171875,
+          "logprob": -10.1015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.1640625,
+          "logprob": -10.1796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.234375,
+          "logprob": -10.2421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.3984375,
+          "logprob": -10.4609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.015625,
+          "logprob": -14.34375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0859375,
+          "logprob": -13.1796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2734375,
+          "logprob": -13.328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.359375,
+          "logprob": -14.3515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0390625,
+          "logprob": -11.0546875,
           "text": "<image>"
         },
         {
@@ -44509,12 +29715,12 @@
         },
         {
           "id": 32000,
-          "logprob": -10.5234375,
+          "logprob": -10.515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.4765625,
+          "logprob": -10.4453125,
           "text": "<image>"
         },
         {
@@ -44524,22 +29730,22 @@
         },
         {
           "id": 32000,
-          "logprob": -13.6171875,
+          "logprob": -13.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.359375,
+          "logprob": -11.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.8359375,
+          "logprob": -10.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -17.34375,
+          "logprob": -17.234375,
           "text": "<image>"
         },
         {
@@ -44554,12 +29760,12 @@
         },
         {
           "id": 32000,
-          "logprob": -10.640625,
+          "logprob": -10.6484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -18.390625,
+          "logprob": -17.984375,
           "text": "<image>"
         },
         {
@@ -44569,17 +29775,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.5625,
+          "logprob": -11.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -9.875,
+          "logprob": -9.8671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.7734375,
+          "logprob": -10.7578125,
           "text": "<image>"
         },
         {
@@ -44589,32 +29795,32 @@
         },
         {
           "id": 32000,
-          "logprob": -10.96875,
+          "logprob": -10.984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.609375,
+          "logprob": -10.625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.09375,
+          "logprob": -11.109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.5078125,
+          "logprob": -10.5,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4453125,
+          "logprob": -11.4609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.59375,
+          "logprob": -13.09375,
           "text": "<image>"
         },
         {
@@ -44624,132 +29830,87 @@
         },
         {
           "id": 32000,
-          "logprob": -10.5625,
+          "logprob": -10.5390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.640625,
+          "logprob": -10.625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9765625,
+          "logprob": -10.9296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.765625,
+          "logprob": -11.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3671875,
+          "logprob": -12.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0234375,
+          "logprob": -12.125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.59375,
+          "logprob": -10.578125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.7421875,
+          "logprob": -10.7265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0625,
+          "logprob": -11.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.3828125,
+          "logprob": -10.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.171875,
+          "logprob": -11.1796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0234375,
+          "logprob": -11.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -18.40625,
+          "logprob": -18.703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9921875,
+          "logprob": -11.015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.7109375,
+          "logprob": -15.6796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.15625,
+          "logprob": -12.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.140625,
+          "logprob": -10.4140625,
           "text": "<image>"
         },
         {
@@ -44759,12 +29920,42 @@
         },
         {
           "id": 32000,
-          "logprob": -10.7890625,
+          "logprob": -11.8046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4296875,
+          "logprob": -10.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.140625,
           "text": "<image>"
         },
         {
@@ -44774,12 +29965,27 @@
         },
         {
           "id": 32000,
-          "logprob": -10.796875,
+          "logprob": -10.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1640625,
+          "logprob": -11.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.15625,
           "text": "<image>"
         },
         {
@@ -44789,27 +29995,27 @@
         },
         {
           "id": 32000,
-          "logprob": -15.4453125,
+          "logprob": -15.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.2109375,
+          "logprob": -10.2265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.09375,
+          "logprob": -11.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.6796875,
+          "logprob": -14.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.3671875,
+          "logprob": -10.375,
           "text": "<image>"
         },
         {
@@ -44824,22 +30030,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.484375,
+          "logprob": -12.4921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.09375,
+          "logprob": -12.1015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.96875,
+          "logprob": -11.0859375,
           "text": "<image>"
         },
         {
@@ -44847,6 +30048,11 @@
           "logprob": -10.9765625,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -10.921875,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -11.0234375,
@@ -44862,11 +30068,6 @@
           "logprob": -11.1484375,
           "text": "<image>"
         },
-        {
-          "id": 32000,
-          "logprob": -10.703125,
-          "text": "<image>"
-        },
         {
           "id": 32000,
           "logprob": -10.71875,
@@ -44874,17 +30075,22 @@
         },
         {
           "id": 32000,
-          "logprob": -11.6875,
+          "logprob": -10.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.8984375,
+          "logprob": -11.6796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2890625,
+          "logprob": -10.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.375,
           "text": "<image>"
         },
         {
@@ -44894,12 +30100,12 @@
         },
         {
           "id": 32000,
-          "logprob": -10.640625,
+          "logprob": -10.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.7109375,
+          "logprob": -10.7265625,
           "text": "<image>"
         },
         {
@@ -44914,17 +30120,17 @@
         },
         {
           "id": 32000,
-          "logprob": -10.6875,
+          "logprob": -10.6796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.84375,
+          "logprob": -11.828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5078125,
+          "logprob": -11.5390625,
           "text": "<image>"
         },
         {
@@ -44934,22 +30140,22 @@
         },
         {
           "id": 32000,
-          "logprob": -10.9609375,
+          "logprob": -10.9765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4921875,
+          "logprob": -12.5,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5546875,
+          "logprob": -11.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.2265625,
+          "logprob": -11.2421875,
           "text": "<image>"
         },
         {
@@ -44959,7 +30165,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.21875,
+          "logprob": -11.2265625,
           "text": "<image>"
         },
         {
@@ -44969,52 +30175,52 @@
         },
         {
           "id": 32000,
-          "logprob": -11.0,
+          "logprob": -10.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0234375,
+          "logprob": -12.046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.09375,
+          "logprob": -11.109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.046875,
+          "logprob": -14.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.921875,
+          "logprob": -10.8984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9609375,
+          "logprob": -10.953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9140625,
+          "logprob": -10.9296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8125,
+          "logprob": -11.78125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.140625,
+          "logprob": -15.1484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9609375,
+          "logprob": -10.96875,
           "text": "<image>"
         },
         {
@@ -45024,72 +30230,72 @@
         },
         {
           "id": 32000,
-          "logprob": -10.8828125,
+          "logprob": -10.890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9609375,
+          "logprob": -10.9453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.46875,
+          "logprob": -11.453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5234375,
+          "logprob": -13.09375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.328125,
+          "logprob": -12.484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.375,
+          "logprob": -11.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3515625,
+          "logprob": -12.3359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.578125,
+          "logprob": -11.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5078125,
+          "logprob": -12.5,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.640625,
+          "logprob": -12.6328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1953125,
+          "logprob": -11.203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9921875,
+          "logprob": -11.9765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.921875,
+          "logprob": -10.9296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.7578125,
+          "logprob": -10.7421875,
           "text": "<image>"
         },
         {
@@ -45099,12 +30305,12 @@
         },
         {
           "id": 32000,
-          "logprob": -15.015625,
+          "logprob": -15.0234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7734375,
+          "logprob": -12.75,
           "text": "<image>"
         },
         {
@@ -45119,27 +30325,27 @@
         },
         {
           "id": 32000,
-          "logprob": -11.2890625,
+          "logprob": -11.265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.34375,
+          "logprob": -13.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.6953125,
+          "logprob": -10.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.59375,
+          "logprob": -10.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6015625,
+          "logprob": -11.7578125,
           "text": "<image>"
         },
         {
@@ -45154,7 +30360,7 @@
         },
         {
           "id": 32000,
-          "logprob": -10.5390625,
+          "logprob": -10.5859375,
           "text": "<image>"
         },
         {
@@ -45164,12 +30370,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.9765625,
+          "logprob": -17.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4609375,
+          "logprob": -11.4765625,
           "text": "<image>"
         },
         {
@@ -45179,12 +30385,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.015625,
+          "logprob": -12.0078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.84375,
+          "logprob": -12.8359375,
           "text": "<image>"
         },
         {
@@ -45194,12 +30400,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.4375,
+          "logprob": -12.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.671875,
+          "logprob": -12.6875,
           "text": "<image>"
         },
         {
@@ -45209,57 +30415,27 @@
         },
         {
           "id": 32000,
-          "logprob": -12.875,
+          "logprob": -12.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2578125,
+          "logprob": -12.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.359375,
+          "logprob": -10.3359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.375,
+          "logprob": -13.46875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.7421875,
+          "logprob": -11.8671875,
           "text": "<image>"
         },
         {
@@ -45269,7 +30445,32 @@
         },
         {
           "id": 32000,
-          "logprob": -11.0390625,
+          "logprob": -11.0078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8203125,
           "text": "<image>"
         },
         {
@@ -45279,17 +30480,22 @@
         },
         {
           "id": 32000,
-          "logprob": -10.6171875,
+          "logprob": -10.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2421875,
+          "logprob": -10.6328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.8359375,
+          "logprob": -12.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8203125,
           "text": "<image>"
         },
         {
@@ -45309,127 +30515,62 @@
         },
         {
           "id": 32000,
-          "logprob": -11.0,
+          "logprob": -10.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7734375,
+          "logprob": -12.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2109375,
+          "logprob": -12.21875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4296875,
+          "logprob": -11.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2421875,
+          "logprob": -12.828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3046875,
+          "logprob": -11.328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.2890625,
+          "logprob": -10.296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.8203125,
+          "logprob": -10.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9140625,
+          "logprob": -11.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.2421875,
+          "logprob": -11.21875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.234375,
+          "logprob": -11.2109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4375,
+          "logprob": -11.5078125,
           "text": "<image>"
         },
         {
@@ -45439,27 +30580,12 @@
         },
         {
           "id": 32000,
-          "logprob": -14.203125,
+          "logprob": -10.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.59375,
+          "logprob": -10.3515625,
           "text": "<image>"
         },
         {
@@ -45469,102 +30595,22 @@
         },
         {
           "id": 32000,
-          "logprob": -10.90625,
+          "logprob": -10.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.2578125,
+          "logprob": -10.828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.921875,
+          "logprob": -11.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.890625,
+          "logprob": -11.9375,
           "text": "<image>"
         },
         {
@@ -45574,112 +30620,32 @@
         },
         {
           "id": 32000,
-          "logprob": -10.9609375,
+          "logprob": -12.3046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.7890625,
+          "logprob": -11.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4453125,
+          "logprob": -12.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.390625,
+          "logprob": -13.3984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1640625,
+          "logprob": -11.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0703125,
+          "logprob": -14.1875,
           "text": "<image>"
         },
         {
@@ -45689,72 +30655,162 @@
         },
         {
           "id": 32000,
-          "logprob": -12.859375,
+          "logprob": -12.421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1875,
+          "logprob": -10.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.171875,
+          "logprob": -10.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6640625,
+          "logprob": -10.6015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1953125,
+          "logprob": -10.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1328125,
+          "logprob": -11.2265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1875,
+          "logprob": -10.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9453125,
+          "logprob": -10.9765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8515625,
+          "logprob": -12.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.203125,
+          "logprob": -10.8671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1796875,
+          "logprob": -10.8515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9609375,
+          "logprob": -11.8984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.703125,
+          "logprob": -10.7421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8515625,
+          "logprob": -13.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1796875,
           "text": "<image>"
         },
         {
@@ -45762,11 +30818,161 @@
           "logprob": -11.75,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -10.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.640625,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -11.8359375,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -11.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.828125,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -12.1953125,
@@ -45774,32 +30980,32 @@
         },
         {
           "id": 32000,
-          "logprob": -12.5078125,
+          "logprob": -12.4921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.546875,
+          "logprob": -11.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.078125,
+          "logprob": -12.0859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.2421875,
+          "logprob": -11.265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6640625,
+          "logprob": -11.6953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2109375,
+          "logprob": -12.21875,
           "text": "<image>"
         },
         {
@@ -45819,27 +31025,7 @@
         },
         {
           "id": 32000,
-          "logprob": -15.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3046875,
+          "logprob": -15.03125,
           "text": "<image>"
         },
         {
@@ -45849,32 +31035,52 @@
         },
         {
           "id": 32000,
-          "logprob": -12.2265625,
+          "logprob": -11.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9453125,
+          "logprob": -13.1171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4921875,
+          "logprob": -12.296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.8828125,
+          "logprob": -12.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3125,
+          "logprob": -12.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8984375,
+          "logprob": -12.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9140625,
           "text": "<image>"
         },
         {
@@ -45884,12 +31090,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8125,
+          "logprob": -11.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.90625,
+          "logprob": -11.9140625,
           "text": "<image>"
         },
         {
@@ -45909,12 +31115,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.140625,
+          "logprob": -13.1484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9765625,
+          "logprob": -11.9921875,
           "text": "<image>"
         },
         {
@@ -45929,7 +31135,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.84375,
+          "logprob": -11.8359375,
           "text": "<image>"
         },
         {
@@ -45944,17 +31150,22 @@
         },
         {
           "id": 32000,
-          "logprob": -13.1953125,
+          "logprob": -13.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.875,
+          "logprob": -11.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.6015625,
+          "logprob": -13.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6484375,
           "text": "<image>"
         },
         {
@@ -45964,27 +31175,22 @@
         },
         {
           "id": 32000,
-          "logprob": -13.671875,
+          "logprob": -11.4609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5390625,
+          "logprob": -13.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.59375,
+          "logprob": -12.4375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5703125,
+          "logprob": -14.6328125,
           "text": "<image>"
         },
         {
@@ -45994,57 +31200,57 @@
         },
         {
           "id": 32000,
-          "logprob": -13.4140625,
+          "logprob": -13.3984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7890625,
+          "logprob": -11.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6328125,
+          "logprob": -11.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.4296875,
+          "logprob": -14.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.53125,
+          "logprob": -13.5390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.515625,
+          "logprob": -14.5078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7265625,
+          "logprob": -12.7109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.609375,
+          "logprob": -11.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.171875,
+          "logprob": -13.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.109375,
+          "logprob": -13.09375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8828125,
+          "logprob": -12.875,
           "text": "<image>"
         },
         {
@@ -46054,37 +31260,37 @@
         },
         {
           "id": 32000,
-          "logprob": -12.671875,
+          "logprob": -12.6484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.7109375,
+          "logprob": -13.6953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4296875,
+          "logprob": -12.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.296875,
+          "logprob": -12.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1796875,
+          "logprob": -13.109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2421875,
+          "logprob": -12.1796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.828125,
+          "logprob": -14.875,
           "text": "<image>"
         },
         {
@@ -46094,7 +31300,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.3359375,
+          "logprob": -11.34375,
           "text": "<image>"
         },
         {
@@ -46114,67 +31320,7 @@
         },
         {
           "id": 32000,
-          "logprob": -15.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0,
+          "logprob": -15.0859375,
           "text": "<image>"
         },
         {
@@ -46184,57 +31330,117 @@
         },
         {
           "id": 32000,
-          "logprob": -16.21875,
+          "logprob": -11.875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6015625,
+          "logprob": -13.25,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.796875,
+          "logprob": -13.5078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7265625,
+          "logprob": -13.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9453125,
+          "logprob": -14.265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9765625,
+          "logprob": -14.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.71875,
+          "logprob": -13.2421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.6953125,
+          "logprob": -12.28125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3359375,
+          "logprob": -13.8046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3203125,
+          "logprob": -12.703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6328125,
+          "logprob": -12.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.609375,
           "text": "<image>"
         },
         {
@@ -46244,17 +31450,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1875,
+          "logprob": -14.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.046875,
+          "logprob": -12.0234375,
           "text": "<image>"
         },
         {
@@ -46269,17 +31475,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.8828125,
+          "logprob": -12.890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.296875,
+          "logprob": -12.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2109375,
+          "logprob": -12.21875,
           "text": "<image>"
         },
         {
@@ -46289,27 +31495,27 @@
         },
         {
           "id": 32000,
-          "logprob": -15.1875,
+          "logprob": -15.1953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5390625,
+          "logprob": -11.546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.421875,
+          "logprob": -14.4375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2890625,
+          "logprob": -12.265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2265625,
+          "logprob": -12.21875,
           "text": "<image>"
         },
         {
@@ -46324,12 +31530,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.5859375,
+          "logprob": -13.5390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.859375,
+          "logprob": -11.8515625,
           "text": "<image>"
         },
         {
@@ -46339,12 +31545,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.1015625,
+          "logprob": -13.046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.9453125,
+          "logprob": -13.953125,
           "text": "<image>"
         },
         {
@@ -46359,47 +31565,47 @@
         },
         {
           "id": 32000,
-          "logprob": -12.734375,
+          "logprob": -12.7109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.203125,
+          "logprob": -12.1796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.59375,
+          "logprob": -12.578125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3984375,
+          "logprob": -11.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5,
+          "logprob": -13.4609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.765625,
+          "logprob": -11.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2265625,
+          "logprob": -13.21875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7578125,
+          "logprob": -11.75,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.3515625,
+          "logprob": -15.34375,
           "text": "<image>"
         },
         {
@@ -46414,22 +31620,42 @@
         },
         {
           "id": 32000,
-          "logprob": -13.3671875,
+          "logprob": -13.3515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.90625,
+          "logprob": -11.8828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5625,
+          "logprob": -12.578125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3203125,
+          "logprob": -13.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.125,
           "text": "<image>"
         },
         {
@@ -46439,27 +31665,7 @@
         },
         {
           "id": 32000,
-          "logprob": -10.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9921875,
+          "logprob": -11.984375,
           "text": "<image>"
         },
         {
@@ -46474,27 +31680,27 @@
         },
         {
           "id": 32000,
-          "logprob": -12.9765625,
+          "logprob": -12.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4921875,
+          "logprob": -12.484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0625,
+          "logprob": -12.046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1796875,
+          "logprob": -12.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3359375,
+          "logprob": -13.296875,
           "text": "<image>"
         },
         {
@@ -46504,32 +31710,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.375,
+          "logprob": -12.3671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.984375,
+          "logprob": -13.1328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9453125,
+          "logprob": -13.625,
           "text": "<image>"
         },
         {
@@ -46539,87 +31730,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.3203125,
+          "logprob": -11.75,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3125,
+          "logprob": -11.9140625,
           "text": "<image>"
         },
         {
@@ -46629,12 +31745,102 @@
         },
         {
           "id": 32000,
-          "logprob": -11.3984375,
+          "logprob": -12.3359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.84375,
+          "logprob": -12.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8359375,
           "text": "<image>"
         },
         {
@@ -46644,52 +31850,52 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6328125,
+          "logprob": -12.6171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.875,
+          "logprob": -13.84375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.109375,
+          "logprob": -12.09375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1171875,
+          "logprob": -12.140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4921875,
+          "logprob": -11.5078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.2890625,
+          "logprob": -11.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.15625,
+          "logprob": -12.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.59375,
+          "logprob": -13.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8046875,
+          "logprob": -11.7890625,
           "text": "<image>"
         },
         {
@@ -46699,22 +31905,22 @@
         },
         {
           "id": 32000,
-          "logprob": -13.1015625,
+          "logprob": -13.0625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2265625,
+          "logprob": -13.1875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2109375,
+          "logprob": -13.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4609375,
+          "logprob": -12.46875,
           "text": "<image>"
         },
         {
@@ -46724,12 +31930,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
+          "logprob": -12.6953125,
           "text": "<image>"
         },
         {
@@ -46737,6 +31938,11 @@
           "logprob": -11.6875,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -11.6953125,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -11.7109375,
@@ -46744,27 +31950,27 @@
         },
         {
           "id": 32000,
-          "logprob": -15.390625,
+          "logprob": -15.3359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.953125,
+          "logprob": -11.9453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.140625,
+          "logprob": -16.125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.4921875,
+          "logprob": -15.4609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.9296875,
+          "logprob": -13.921875,
           "text": "<image>"
         },
         {
@@ -46779,7 +31985,7 @@
         },
         {
           "id": 32000,
-          "logprob": -15.984375,
+          "logprob": -16.0,
           "text": "<image>"
         },
         {
@@ -46789,37 +31995,37 @@
         },
         {
           "id": 32000,
-          "logprob": -12.8671875,
+          "logprob": -12.828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4921875,
+          "logprob": -12.5078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7421875,
+          "logprob": -11.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1875,
+          "logprob": -14.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3515625,
+          "logprob": -11.34375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.71875,
+          "logprob": -11.7265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0625,
+          "logprob": -13.046875,
           "text": "<image>"
         },
         {
@@ -46829,12 +32035,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.421875,
+          "logprob": -12.4375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.2734375,
+          "logprob": -14.265625,
           "text": "<image>"
         },
         {
@@ -46844,77 +32050,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.390625,
+          "logprob": -12.6953125,
           "text": "<image>"
         },
         {
@@ -46924,27 +32060,97 @@
         },
         {
           "id": 32000,
-          "logprob": -13.078125,
+          "logprob": -12.984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4296875,
+          "logprob": -13.2890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.828125,
+          "logprob": -13.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8046875,
+          "logprob": -11.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6484375,
+          "logprob": -10.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.625,
           "text": "<image>"
         },
         {
@@ -46959,27 +32165,27 @@
         },
         {
           "id": 32000,
-          "logprob": -14.921875,
+          "logprob": -14.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.578125,
+          "logprob": -12.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.78125,
+          "logprob": -12.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3984375,
+          "logprob": -11.375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.0546875,
+          "logprob": -14.03125,
           "text": "<image>"
         },
         {
@@ -46999,12 +32205,12 @@
         },
         {
           "id": 32000,
-          "logprob": -14.5234375,
+          "logprob": -14.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.625,
+          "logprob": -12.609375,
           "text": "<image>"
         },
         {
@@ -47014,12 +32220,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.6015625,
+          "logprob": -13.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.28125,
+          "logprob": -13.2109375,
           "text": "<image>"
         },
         {
@@ -47039,17 +32245,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.0,
+          "logprob": -12.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6640625,
+          "logprob": -12.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.46875,
+          "logprob": -11.484375,
           "text": "<image>"
         },
         {
@@ -47059,12 +32265,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8828125,
+          "logprob": -11.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1015625,
+          "logprob": -13.078125,
           "text": "<image>"
         },
         {
@@ -47079,7 +32285,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.5234375,
+          "logprob": -11.546875,
           "text": "<image>"
         },
         {
@@ -47094,7 +32300,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.4921875,
+          "logprob": -12.5,
           "text": "<image>"
         },
         {
@@ -47109,122 +32315,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.7890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1484375,
+          "logprob": -12.078125,
           "text": "<image>"
         },
         {
@@ -47234,132 +32325,102 @@
         },
         {
           "id": 32000,
-          "logprob": -13.3828125,
+          "logprob": -13.2265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3203125,
+          "logprob": -14.8046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.34375,
+          "logprob": -12.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -11.8671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.140625,
+          "logprob": -13.1484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9921875,
+          "logprob": -12.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.15625,
+          "logprob": -10.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.84375,
+          "logprob": -14.6875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.21875,
+          "logprob": -15.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.46875,
+          "logprob": -15.453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1484375,
+          "logprob": -11.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.515625,
+          "logprob": -14.484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.84375,
+          "logprob": -12.4375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.015625,
+          "logprob": -16.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.28125,
+          "logprob": -14.875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6015625,
+          "logprob": -16.109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3984375,
+          "logprob": -13.2890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.375,
+          "logprob": -12.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5625,
+          "logprob": -11.8359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -9.9375,
+          "logprob": -12.9921875,
           "text": "<image>"
         },
         {
@@ -47369,17 +32430,162 @@
         },
         {
           "id": 32000,
-          "logprob": -12.203125,
+          "logprob": -12.1484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.2890625,
+          "logprob": -13.84375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1953125,
+          "logprob": -13.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -9.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.15625,
           "text": "<image>"
         },
         {
@@ -47392,21 +32598,6 @@
           "logprob": -12.28125,
           "text": "<image>"
         },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
         {
           "id": 32000,
           "logprob": -11.8984375,
@@ -47414,52 +32605,67 @@
         },
         {
           "id": 32000,
-          "logprob": -10.5859375,
+          "logprob": -14.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9921875,
+          "logprob": -14.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9921875,
+          "logprob": -11.8671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1015625,
+          "logprob": -10.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.5390625,
+          "logprob": -13.0,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.2578125,
+          "logprob": -12.8984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1171875,
+          "logprob": -12.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2421875,
+          "logprob": -15.765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5,
+          "logprob": -15.203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7265625,
+          "logprob": -14.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8359375,
           "text": "<image>"
         },
         {
@@ -47469,7 +32675,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.0,
+          "logprob": -11.0078125,
           "text": "<image>"
         },
         {
@@ -47479,22 +32685,22 @@
         },
         {
           "id": 32000,
-          "logprob": -13.28125,
+          "logprob": -13.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7734375,
+          "logprob": -11.7578125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0625,
+          "logprob": -13.09375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.671875,
+          "logprob": -13.6796875,
           "text": "<image>"
         },
         {
@@ -47504,7 +32710,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.3828125,
+          "logprob": -12.421875,
           "text": "<image>"
         },
         {
@@ -47514,22 +32720,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.25,
+          "logprob": -12.2890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.9453125,
+          "logprob": -11.921875,
           "text": "<image>"
         },
         {
@@ -47539,7 +32735,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.9375,
+          "logprob": -15.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
           "text": "<image>"
         },
         {
@@ -47549,37 +32755,37 @@
         },
         {
           "id": 32000,
-          "logprob": -11.3203125,
+          "logprob": -11.328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4921875,
+          "logprob": -11.5,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.359375,
+          "logprob": -11.3671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3359375,
+          "logprob": -11.3515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0546875,
+          "logprob": -12.0625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.359375,
+          "logprob": -12.3359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -9.6953125,
+          "logprob": -9.7109375,
           "text": "<image>"
         },
         {
@@ -47589,27 +32795,27 @@
         },
         {
           "id": 32000,
-          "logprob": -14.3203125,
+          "logprob": -14.2421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9609375,
+          "logprob": -11.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0859375,
+          "logprob": -12.09375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2109375,
+          "logprob": -12.1796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4921875,
+          "logprob": -11.5234375,
           "text": "<image>"
         },
         {
@@ -47624,7 +32830,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.0546875,
           "text": "<image>"
         },
         {
@@ -47639,22 +32845,22 @@
         },
         {
           "id": 32000,
-          "logprob": -11.9921875,
+          "logprob": -12.0,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2265625,
+          "logprob": -12.1953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9921875,
+          "logprob": -12.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6796875,
+          "logprob": -12.7109375,
           "text": "<image>"
         },
         {
@@ -47664,7 +32870,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.5703125,
+          "logprob": -13.5546875,
           "text": "<image>"
         },
         {
@@ -47679,17 +32885,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.9375,
+          "logprob": -11.953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.9453125,
+          "logprob": -13.8046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.984375,
+          "logprob": -11.9921875,
           "text": "<image>"
         },
         {
@@ -47699,17 +32905,17 @@
         },
         {
           "id": 32000,
-          "logprob": -10.03125,
+          "logprob": -10.015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7265625,
+          "logprob": -11.71875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.7890625,
+          "logprob": -13.8125,
           "text": "<image>"
         },
         {
@@ -47719,42 +32925,42 @@
         },
         {
           "id": 32000,
-          "logprob": -12.671875,
+          "logprob": -12.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.59375,
+          "logprob": -12.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1171875,
+          "logprob": -12.1015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4609375,
+          "logprob": -12.453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3046875,
+          "logprob": -12.2890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4765625,
+          "logprob": -12.46875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6328125,
+          "logprob": -12.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.03125,
+          "logprob": -12.015625,
           "text": "<image>"
         },
         {
@@ -47764,132 +32970,32 @@
         },
         {
           "id": 32000,
-          "logprob": -14.5078125,
+          "logprob": -14.421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2265625,
+          "logprob": -13.25,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.546875,
+          "logprob": -16.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.4765625,
+          "logprob": -14.484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.71875,
+          "logprob": -13.7421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.7890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.671875,
+          "logprob": -13.46875,
           "text": "<image>"
         },
         {
@@ -47899,57 +33005,97 @@
         },
         {
           "id": 32000,
-          "logprob": -13.40625,
+          "logprob": -12.8671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.265625,
+          "logprob": -12.4921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.015625,
+          "logprob": -13.2265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5390625,
+          "logprob": -13.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.0234375,
+          "logprob": -12.4375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1796875,
+          "logprob": -10.765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6875,
+          "logprob": -13.109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.9453125,
+          "logprob": -12.8515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6875,
+          "logprob": -14.7421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.78125,
+          "logprob": -12.84375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.9609375,
+          "logprob": -12.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6484375,
           "text": "<image>"
         },
         {
@@ -47959,27 +33105,87 @@
         },
         {
           "id": 32000,
-          "logprob": -16.140625,
+          "logprob": -13.4296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4609375,
+          "logprob": -13.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.7265625,
+          "logprob": -12.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.609375,
+          "logprob": -11.5234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.234375,
+          "logprob": -15.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.25,
           "text": "<image>"
         },
         {
@@ -47989,7 +33195,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.4921875,
+          "logprob": -13.5,
           "text": "<image>"
         },
         {
@@ -48014,12 +33220,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.15625,
+          "logprob": -13.1484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -17.046875,
+          "logprob": -17.0625,
           "text": "<image>"
         },
         {
@@ -48029,12 +33235,12 @@
         },
         {
           "id": 32000,
-          "logprob": -14.265625,
+          "logprob": -14.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6328125,
+          "logprob": -12.625,
           "text": "<image>"
         },
         {
@@ -48044,92 +33250,37 @@
         },
         {
           "id": 32000,
-          "logprob": -13.9375,
+          "logprob": -13.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0390625,
+          "logprob": -13.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.46875,
+          "logprob": -10.5,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1640625,
+          "logprob": -13.1953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.59375,
+          "logprob": -13.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.390625,
+          "logprob": -13.3828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8125,
+          "logprob": -14.0625,
           "text": "<image>"
         },
         {
@@ -48137,6 +33288,61 @@
           "logprob": -13.5234375,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -14.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.53125,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -11.4140625,
@@ -48144,17 +33350,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.46875,
+          "logprob": -11.4609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.09375,
+          "logprob": -13.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.34375,
+          "logprob": -12.3359375,
           "text": "<image>"
         },
         {
@@ -48164,32 +33370,32 @@
         },
         {
           "id": 32000,
-          "logprob": -12.7578125,
+          "logprob": -12.75,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.8359375,
+          "logprob": -14.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0859375,
+          "logprob": -13.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1640625,
+          "logprob": -14.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0625,
+          "logprob": -13.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.5078125,
+          "logprob": -14.4921875,
           "text": "<image>"
         },
         {
@@ -48199,7 +33405,7 @@
         },
         {
           "id": 32000,
-          "logprob": -14.6953125,
+          "logprob": -14.703125,
           "text": "<image>"
         },
         {
@@ -48209,32 +33415,32 @@
         },
         {
           "id": 32000,
-          "logprob": -16.328125,
+          "logprob": -16.3125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.9921875,
+          "logprob": -14.9765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.8515625,
+          "logprob": -13.8046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.40625,
+          "logprob": -15.375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4296875,
+          "logprob": -13.3984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.5859375,
+          "logprob": -14.578125,
           "text": "<image>"
         },
         {
@@ -48249,7 +33455,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.3359375,
+          "logprob": -13.328125,
           "text": "<image>"
         },
         {
@@ -48264,37 +33470,52 @@
         },
         {
           "id": 32000,
-          "logprob": -16.890625,
+          "logprob": -16.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.328125,
+          "logprob": -13.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.78125,
+          "logprob": -13.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.34375,
+          "logprob": -13.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4921875,
+          "logprob": -13.4765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4296875,
+          "logprob": -13.390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.28125,
+          "logprob": -13.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5,
           "text": "<image>"
         },
         {
@@ -48304,7 +33525,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.75,
+          "logprob": -13.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6796875,
           "text": "<image>"
         },
         {
@@ -48314,32 +33545,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.328125,
+          "logprob": -13.3203125,
           "text": "<image>"
         },
         {
@@ -48349,47 +33555,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.640625,
+          "logprob": -13.6171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.15625,
+          "logprob": -15.140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0078125,
+          "logprob": -13.21875,
           "text": "<image>"
         },
         {
@@ -48399,27 +33575,57 @@
         },
         {
           "id": 32000,
-          "logprob": -11.625,
+          "logprob": -12.0234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8671875,
+          "logprob": -11.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.578125,
+          "logprob": -11.3125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8125,
+          "logprob": -10.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.9453125,
+          "logprob": -13.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.9765625,
           "text": "<image>"
         },
         {
@@ -48429,17 +33635,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.9765625,
+          "logprob": -12.984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.828125,
+          "logprob": -13.8359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7265625,
+          "logprob": -12.75,
           "text": "<image>"
         },
         {
@@ -48454,12 +33660,12 @@
         },
         {
           "id": 32000,
-          "logprob": -14.7578125,
+          "logprob": -14.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.5234375,
+          "logprob": -14.5078125,
           "text": "<image>"
         },
         {
@@ -48469,12 +33675,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.0390625,
+          "logprob": -13.046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4453125,
+          "logprob": -13.453125,
           "text": "<image>"
         },
         {
@@ -48489,17 +33695,17 @@
         },
         {
           "id": 32000,
-          "logprob": -14.0,
+          "logprob": -13.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5390625,
+          "logprob": -13.4921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0859375,
+          "logprob": -11.9765625,
           "text": "<image>"
         },
         {
@@ -48509,17 +33715,37 @@
         },
         {
           "id": 32000,
-          "logprob": -15.953125,
+          "logprob": -15.9609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.2265625,
+          "logprob": -14.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1484375,
+          "logprob": -13.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.640625,
           "text": "<image>"
         },
         {
@@ -48529,32 +33755,47 @@
         },
         {
           "id": 32000,
-          "logprob": -12.6875,
+          "logprob": -13.625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.796875,
+          "logprob": -12.890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.65625,
+          "logprob": -12.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6796875,
+          "logprob": -12.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.65625,
+          "logprob": -12.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.90625,
+          "logprob": -14.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0078125,
           "text": "<image>"
         },
         {
@@ -48564,52 +33805,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.6171875,
+          "logprob": -15.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7890625,
+          "logprob": -12.921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.375,
+          "logprob": -12.40625,
           "text": "<image>"
         },
         {
@@ -48624,27 +33830,27 @@
         },
         {
           "id": 32000,
-          "logprob": -11.375,
+          "logprob": -11.3671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0625,
+          "logprob": -11.0703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3046875,
+          "logprob": -11.28125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0390625,
+          "logprob": -11.046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3515625,
+          "logprob": -13.3203125,
           "text": "<image>"
         },
         {
@@ -48669,12 +33875,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.8359375,
+          "logprob": -13.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0234375,
+          "logprob": -12.0859375,
           "text": "<image>"
         },
         {
@@ -48684,12 +33890,7 @@
         },
         {
           "id": 32000,
-          "logprob": -14.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0546875,
+          "logprob": -14.0703125,
           "text": "<image>"
         },
         {
@@ -48699,7 +33900,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.0625,
+          "logprob": -14.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0390625,
           "text": "<image>"
         },
         {
@@ -48709,42 +33915,42 @@
         },
         {
           "id": 32000,
-          "logprob": -14.46875,
+          "logprob": -14.453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.703125,
+          "logprob": -14.6875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.296875,
+          "logprob": -14.3046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.8828125,
+          "logprob": -14.8671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1796875,
+          "logprob": -14.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2265625,
+          "logprob": -13.203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1171875,
+          "logprob": -13.109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9375,
+          "logprob": -11.8515625,
           "text": "<image>"
         },
         {
@@ -48759,12 +33965,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.71875,
+          "logprob": -12.7109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.734375,
+          "logprob": -13.7109375,
           "text": "<image>"
         },
         {
@@ -48774,32 +33980,32 @@
         },
         {
           "id": 32000,
-          "logprob": -12.46875,
+          "logprob": -12.4765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.15625,
+          "logprob": -12.140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.375,
+          "logprob": -12.3828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8984375,
+          "logprob": -12.875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.6953125,
+          "logprob": -13.6484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0625,
+          "logprob": -13.0390625,
           "text": "<image>"
         },
         {
@@ -48809,12 +34015,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.6796875,
+          "logprob": -12.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3515625,
+          "logprob": -12.34375,
           "text": "<image>"
         },
         {
@@ -48824,37 +34030,37 @@
         },
         {
           "id": 32000,
-          "logprob": -12.46875,
+          "logprob": -12.453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.03125,
+          "logprob": -12.015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.96875,
+          "logprob": -11.9765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.4609375,
+          "logprob": -14.46875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.328125,
+          "logprob": -13.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6484375,
+          "logprob": -12.71875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.65625,
+          "logprob": -10.6484375,
           "text": "<image>"
         },
         {
@@ -48874,22 +34080,22 @@
         },
         {
           "id": 32000,
-          "logprob": -12.6328125,
+          "logprob": -12.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.265625,
+          "logprob": -11.2578125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.734375,
+          "logprob": -11.7265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.84375,
+          "logprob": -10.8515625,
           "text": "<image>"
         },
         {
@@ -48899,32 +34105,32 @@
         },
         {
           "id": 32000,
-          "logprob": -13.109375,
+          "logprob": -13.140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7421875,
+          "logprob": -12.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.09375,
+          "logprob": -12.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.109375,
+          "logprob": -13.1328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.671875,
+          "logprob": -12.65625,
           "text": "<image>"
         },
         {
@@ -48934,27 +34140,27 @@
         },
         {
           "id": 32000,
-          "logprob": -12.6796875,
+          "logprob": -12.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.109375,
+          "logprob": -12.0859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.625,
+          "logprob": -13.609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8515625,
+          "logprob": -12.8359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3046875,
+          "logprob": -12.3125,
           "text": "<image>"
         },
         {
@@ -48964,12 +34170,12 @@
         },
         {
           "id": 32000,
-          "logprob": -14.0703125,
+          "logprob": -14.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.0,
+          "logprob": -13.984375,
           "text": "<image>"
         },
         {
@@ -48979,7 +34185,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.2421875,
+          "logprob": -13.2109375,
           "text": "<image>"
         },
         {
@@ -48989,42 +34195,42 @@
         },
         {
           "id": 32000,
-          "logprob": -13.7109375,
+          "logprob": -13.703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6015625,
+          "logprob": -12.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.796875,
+          "logprob": -15.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.90625,
+          "logprob": -11.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.375,
+          "logprob": -12.3671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4765625,
+          "logprob": -11.46875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.015625,
+          "logprob": -12.0,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.328125,
+          "logprob": -11.3359375,
           "text": "<image>"
         },
         {
@@ -49039,17 +34245,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.8203125,
+          "logprob": -13.828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7578125,
+          "logprob": -11.75,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.484375,
+          "logprob": -11.46875,
           "text": "<image>"
         },
         {
@@ -49064,17 +34270,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.640625,
+          "logprob": -11.625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5859375,
+          "logprob": -11.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.2578125,
+          "logprob": -15.2265625,
           "text": "<image>"
         },
         {
@@ -49084,17 +34290,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.0078125,
+          "logprob": -13.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.3828125,
+          "logprob": -15.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.203125,
+          "logprob": -11.1875,
           "text": "<image>"
         },
         {
@@ -49104,7 +34310,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.0703125,
+          "logprob": -11.0625,
           "text": "<image>"
         },
         {
@@ -49114,7 +34320,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.4453125,
+          "logprob": -13.453125,
           "text": "<image>"
         },
         {
@@ -49124,27 +34330,27 @@
         },
         {
           "id": 32000,
-          "logprob": -11.5078125,
+          "logprob": -11.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.015625,
+          "logprob": -12.0078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.15625,
+          "logprob": -11.1875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8359375,
+          "logprob": -11.890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.359375,
+          "logprob": -11.3125,
           "text": "<image>"
         },
         {
@@ -49159,22 +34365,22 @@
         },
         {
           "id": 32000,
-          "logprob": -12.4453125,
+          "logprob": -12.4921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.015625,
+          "logprob": -12.0703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.375,
+          "logprob": -13.3671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2734375,
+          "logprob": -13.28125,
           "text": "<image>"
         },
         {
@@ -49184,22 +34390,22 @@
         },
         {
           "id": 32000,
-          "logprob": -13.046875,
+          "logprob": -13.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0859375,
+          "logprob": -13.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1640625,
+          "logprob": -12.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4765625,
+          "logprob": -13.484375,
           "text": "<image>"
         },
         {
@@ -49209,7 +34415,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.7265625,
+          "logprob": -13.703125,
           "text": "<image>"
         },
         {
@@ -49219,262 +34425,17 @@
         },
         {
           "id": 32000,
-          "logprob": -14.3359375,
+          "logprob": -14.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.71875,
+          "logprob": -12.7109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.15625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.640625,
+          "logprob": -14.28125,
           "text": "<image>"
         },
         {
@@ -49484,52 +34445,57 @@
         },
         {
           "id": 32000,
-          "logprob": -13.453125,
+          "logprob": -13.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.4375,
+          "logprob": -15.203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1171875,
+          "logprob": -13.28125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4296875,
+          "logprob": -12.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.515625,
+          "logprob": -14.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.21875,
+          "logprob": -13.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8671875,
+          "logprob": -12.1171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8515625,
+          "logprob": -14.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.8671875,
+          "logprob": -14.0625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.59375,
+          "logprob": -12.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.953125,
           "text": "<image>"
         },
         {
@@ -49539,12 +34505,57 @@
         },
         {
           "id": 32000,
-          "logprob": -12.1640625,
+          "logprob": -11.6953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2265625,
+          "logprob": -12.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5234375,
           "text": "<image>"
         },
         {
@@ -49554,32 +34565,27 @@
         },
         {
           "id": 32000,
-          "logprob": -12.1015625,
+          "logprob": -11.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0234375,
+          "logprob": -11.2578125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4140625,
+          "logprob": -13.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.828125,
+          "logprob": -11.921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1953125,
+          "logprob": -13.03125,
           "text": "<image>"
         },
         {
@@ -49589,7 +34595,207 @@
         },
         {
           "id": 32000,
-          "logprob": -11.1328125,
+          "logprob": -14.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.4296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.140625,
           "text": "<image>"
         },
         {
@@ -49614,7 +34820,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.1015625,
+          "logprob": -11.1171875,
           "text": "<image>"
         },
         {
@@ -49624,7 +34830,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.15625,
+          "logprob": -11.1484375,
           "text": "<image>"
         },
         {
@@ -49634,17 +34840,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.4296875,
+          "logprob": -11.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.765625,
+          "logprob": -12.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.09375,
+          "logprob": -13.0859375,
           "text": "<image>"
         },
         {
@@ -49654,22 +34860,22 @@
         },
         {
           "id": 32000,
-          "logprob": -12.96875,
+          "logprob": -12.9609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.328125,
+          "logprob": -12.34375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8359375,
+          "logprob": -12.84375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.9609375,
+          "logprob": -13.9765625,
           "text": "<image>"
         },
         {
@@ -49679,7 +34885,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.46875,
+          "logprob": -13.484375,
           "text": "<image>"
         },
         {
@@ -49689,22 +34895,22 @@
         },
         {
           "id": 32000,
-          "logprob": -13.59375,
+          "logprob": -13.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.40625,
+          "logprob": -13.3828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.234375,
+          "logprob": -14.25,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.265625,
+          "logprob": -14.2578125,
           "text": "<image>"
         },
         {
@@ -49714,127 +34920,127 @@
         },
         {
           "id": 32000,
-          "logprob": -13.65625,
+          "logprob": -13.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.2734375,
+          "logprob": -14.2578125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.09375,
+          "logprob": -13.8671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.890625,
+          "logprob": -12.921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.640625,
+          "logprob": -15.6484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0234375,
+          "logprob": -13.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3828125,
+          "logprob": -12.34375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.71875,
+          "logprob": -12.7265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.5234375,
+          "logprob": -14.515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7734375,
+          "logprob": -12.75,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3203125,
+          "logprob": -12.265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.609375,
+          "logprob": -13.546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1640625,
+          "logprob": -13.1328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.203125,
+          "logprob": -12.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1875,
+          "logprob": -12.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.59375,
+          "logprob": -11.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6875,
+          "logprob": -11.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1796875,
+          "logprob": -12.15625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6484375,
+          "logprob": -11.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.421875,
+          "logprob": -11.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.34375,
+          "logprob": -11.390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1015625,
+          "logprob": -11.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9765625,
+          "logprob": -11.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.015625,
+          "logprob": -12.0078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0,
+          "logprob": -12.984375,
           "text": "<image>"
         },
         {
@@ -49844,7 +35050,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.09375,
           "text": "<image>"
         },
         {
@@ -49864,12 +35070,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.65625,
+          "logprob": -11.6484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.84375,
+          "logprob": -12.8125,
           "text": "<image>"
         },
         {
@@ -49879,17 +35085,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.6953125,
+          "logprob": -11.6796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.578125,
+          "logprob": -11.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.6796875,
+          "logprob": -13.6953125,
           "text": "<image>"
         },
         {
@@ -49904,7 +35110,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0078125,
+          "logprob": -12.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.296875,
           "text": "<image>"
         },
         {
@@ -49914,17 +35125,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.234375,
+          "logprob": -13.84375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8671875,
+          "logprob": -12.875,
           "text": "<image>"
         },
         {
@@ -49934,17 +35140,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.5078125,
+          "logprob": -13.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.953125,
+          "logprob": -13.9609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5078125,
+          "logprob": -12.46875,
           "text": "<image>"
         },
         {
@@ -49954,232 +35160,17 @@
         },
         {
           "id": 32000,
-          "logprob": -14.25,
+          "logprob": -14.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9140625,
+          "logprob": -12.921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4375,
+          "logprob": -14.2578125,
           "text": "<image>"
         },
         {
@@ -50189,62 +35180,62 @@
         },
         {
           "id": 32000,
-          "logprob": -13.78125,
+          "logprob": -14.3984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5703125,
+          "logprob": -15.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9765625,
+          "logprob": -13.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4765625,
+          "logprob": -12.3515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.125,
+          "logprob": -13.46875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0859375,
+          "logprob": -11.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9296875,
+          "logprob": -12.2890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.90625,
+          "logprob": -14.3515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0625,
+          "logprob": -13.140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0234375,
+          "logprob": -13.0234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.6953125,
+          "logprob": -12.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.890625,
+          "logprob": -11.8125,
           "text": "<image>"
         },
         {
@@ -50252,6 +35243,31 @@
           "logprob": -12.125,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -11.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.640625,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -12.4375,
@@ -50259,67 +35275,257 @@
         },
         {
           "id": 32000,
-          "logprob": -12.375,
+          "logprob": -11.46875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.28125,
+          "logprob": -11.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4921875,
+          "logprob": -12.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1640625,
+          "logprob": -11.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1953125,
+          "logprob": -11.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.4140625,
+          "logprob": -12.4453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6171875,
+          "logprob": -11.34375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.921875,
+          "logprob": -12.1015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6328125,
+          "logprob": -11.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2421875,
+          "logprob": -14.6015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0234375,
+          "logprob": -12.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6953125,
+          "logprob": -11.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1484375,
+          "logprob": -11.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1328125,
           "text": "<image>"
         },
         {
@@ -50329,27 +35535,27 @@
         },
         {
           "id": 32000,
-          "logprob": -11.7265625,
+          "logprob": -11.71875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6796875,
+          "logprob": -11.6875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4921875,
+          "logprob": -12.484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.875,
+          "logprob": -11.8828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5390625,
+          "logprob": -12.53125,
           "text": "<image>"
         },
         {
@@ -50364,27 +35570,27 @@
         },
         {
           "id": 32000,
-          "logprob": -13.71875,
+          "logprob": -13.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.78125,
+          "logprob": -14.8046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.578125,
+          "logprob": -11.609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5859375,
+          "logprob": -13.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.390625,
+          "logprob": -12.421875,
           "text": "<image>"
         },
         {
@@ -50394,17 +35600,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.65625,
+          "logprob": -13.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5,
+          "logprob": -12.5078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.109375,
+          "logprob": -13.125,
           "text": "<image>"
         },
         {
@@ -50414,37 +35620,37 @@
         },
         {
           "id": 32000,
-          "logprob": -12.2578125,
+          "logprob": -12.2890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7421875,
+          "logprob": -12.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1640625,
+          "logprob": -13.15625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9375,
+          "logprob": -11.9453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.265625,
+          "logprob": -12.28125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1640625,
+          "logprob": -12.1328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.8671875,
+          "logprob": -13.8203125,
           "text": "<image>"
         },
         {
@@ -50454,57 +35660,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.046875,
+          "logprob": -13.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.984375,
+          "logprob": -15.6953125,
           "text": "<image>"
         },
         {
@@ -50514,12 +35675,47 @@
         },
         {
           "id": 32000,
-          "logprob": -11.3984375,
+          "logprob": -14.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.796875,
+          "logprob": -13.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.234375,
           "text": "<image>"
         },
         {
@@ -50529,37 +35725,47 @@
         },
         {
           "id": 32000,
-          "logprob": -11.4296875,
+          "logprob": -11.828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9921875,
+          "logprob": -11.453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.59375,
+          "logprob": -11.453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8359375,
+          "logprob": -11.984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7421875,
+          "logprob": -11.609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.125,
+          "logprob": -11.8046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.875,
+          "logprob": -11.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8515625,
           "text": "<image>"
         },
         {
@@ -50569,27 +35775,27 @@
         },
         {
           "id": 32000,
-          "logprob": -10.796875,
+          "logprob": -10.8046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4765625,
+          "logprob": -11.578125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1484375,
+          "logprob": -11.15625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8046875,
+          "logprob": -12.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3828125,
+          "logprob": -11.3671875,
           "text": "<image>"
         },
         {
@@ -50609,7 +35815,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.5546875,
+          "logprob": -12.5625,
           "text": "<image>"
         },
         {
@@ -50619,367 +35825,57 @@
         },
         {
           "id": 32000,
-          "logprob": -10.40625,
+          "logprob": -10.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.59375,
+          "logprob": -12.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.03125,
+          "logprob": -12.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.03125,
+          "logprob": -12.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.65625,
+          "logprob": -12.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6328125,
+          "logprob": -11.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5,
+          "logprob": -13.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.078125,
+          "logprob": -12.0703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0234375,
+          "logprob": -13.015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.171875,
+          "logprob": -14.1484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.4609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.015625,
+          "logprob": -12.3671875,
           "text": "<image>"
         },
         {
@@ -50989,42 +35885,112 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0859375,
+          "logprob": -12.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.140625,
+          "logprob": -13.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.65625,
+          "logprob": -13.953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.890625,
+          "logprob": -12.4765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.59375,
+          "logprob": -14.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4375,
+          "logprob": -12.6015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4375,
+          "logprob": -13.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.109375,
+          "logprob": -15.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0546875,
           "text": "<image>"
         },
         {
@@ -51034,17 +36000,257 @@
         },
         {
           "id": 32000,
-          "logprob": -11.1015625,
+          "logprob": -11.9609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.203125,
+          "logprob": -14.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9921875,
+          "logprob": -12.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.1484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.96875,
           "text": "<image>"
         },
         {
@@ -51054,32 +36260,32 @@
         },
         {
           "id": 32000,
-          "logprob": -11.40625,
+          "logprob": -11.4296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.53125,
+          "logprob": -12.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.109375,
+          "logprob": -12.1015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3203125,
+          "logprob": -11.3125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.125,
+          "logprob": -11.1171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7578125,
+          "logprob": -11.75,
           "text": "<image>"
         },
         {
@@ -51094,7 +36300,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.109375,
+          "logprob": -11.1015625,
           "text": "<image>"
         },
         {
@@ -51114,67 +36320,67 @@
         },
         {
           "id": 32000,
-          "logprob": -12.609375,
+          "logprob": -12.6015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.65625,
+          "logprob": -12.6796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7890625,
+          "logprob": -12.8359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.28125,
+          "logprob": -12.296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4140625,
+          "logprob": -13.3671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.96875,
+          "logprob": -12.9765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1171875,
+          "logprob": -12.109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5390625,
+          "logprob": -12.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.609375,
+          "logprob": -12.6015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6484375,
+          "logprob": -12.625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.09375,
+          "logprob": -12.109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9921875,
+          "logprob": -13.0,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4453125,
+          "logprob": -13.4296875,
           "text": "<image>"
         },
         {
@@ -51184,17 +36390,17 @@
         },
         {
           "id": 32000,
-          "logprob": -14.7109375,
+          "logprob": -14.71875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.625,
+          "logprob": -13.6015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.765625,
+          "logprob": -13.7578125,
           "text": "<image>"
         },
         {
@@ -51204,37 +36410,37 @@
         },
         {
           "id": 32000,
-          "logprob": -14.53125,
+          "logprob": -14.5390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.34375,
+          "logprob": -13.3515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.921875,
+          "logprob": -12.9453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.359375,
+          "logprob": -11.3515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.078125,
+          "logprob": -12.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4609375,
+          "logprob": -11.421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.640625,
+          "logprob": -11.625,
           "text": "<image>"
         },
         {
@@ -51249,62 +36455,62 @@
         },
         {
           "id": 32000,
-          "logprob": -11.953125,
+          "logprob": -11.9453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0859375,
+          "logprob": -12.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.421875,
+          "logprob": -11.4296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5859375,
+          "logprob": -11.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0859375,
+          "logprob": -12.0703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.25,
+          "logprob": -11.2421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6171875,
+          "logprob": -11.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3359375,
+          "logprob": -11.3125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.46875,
+          "logprob": -11.453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.265625,
+          "logprob": -12.28125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.21875,
+          "logprob": -12.2109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1875,
+          "logprob": -12.203125,
           "text": "<image>"
         },
         {
@@ -51319,47 +36525,47 @@
         },
         {
           "id": 32000,
-          "logprob": -11.2421875,
+          "logprob": -11.25,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3046875,
+          "logprob": -12.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.03125,
+          "logprob": -11.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.203125,
+          "logprob": -11.1953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.125,
+          "logprob": -11.1171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9296875,
+          "logprob": -11.9375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9765625,
+          "logprob": -10.984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.453125,
+          "logprob": -11.390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.859375,
+          "logprob": -12.8828125,
           "text": "<image>"
         },
         {
@@ -51384,47 +36590,47 @@
         },
         {
           "id": 32000,
-          "logprob": -12.1328125,
+          "logprob": -12.1171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5,
+          "logprob": -12.515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8671875,
+          "logprob": -11.890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1640625,
+          "logprob": -13.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.765625,
+          "logprob": -15.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.609375,
+          "logprob": -11.6015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.34375,
+          "logprob": -13.328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.671875,
+          "logprob": -13.6484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.9140625,
+          "logprob": -14.875,
           "text": "<image>"
         },
         {
@@ -51434,57 +36640,87 @@
         },
         {
           "id": 32000,
-          "logprob": -13.53125,
+          "logprob": -13.5234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.25,
+          "logprob": -13.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6171875,
+          "logprob": -12.609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.8046875,
+          "logprob": -13.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.375,
+          "logprob": -14.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.171875,
+          "logprob": -12.1484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.828125,
+          "logprob": -11.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.875,
+          "logprob": -11.890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.484375,
+          "logprob": -11.4921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5625,
+          "logprob": -12.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.640625,
+          "logprob": -11.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.03125,
           "text": "<image>"
         },
         {
@@ -51492,36 +36728,6 @@
           "logprob": -11.7265625,
           "text": "<image>"
         },
-        {
-          "id": 32000,
-          "logprob": -11.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.734375,
-          "text": "<image>"
-        },
         {
           "id": 32000,
           "logprob": -11.84375,
@@ -51529,12 +36735,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0390625,
+          "logprob": -11.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.2578125,
+          "logprob": -11.2421875,
           "text": "<image>"
         },
         {
@@ -51569,22 +36775,22 @@
         },
         {
           "id": 32000,
-          "logprob": -11.0390625,
+          "logprob": -11.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1953125,
+          "logprob": -12.203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.6640625,
+          "logprob": -13.609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6640625,
+          "logprob": -12.65625,
           "text": "<image>"
         },
         {
@@ -51609,27 +36815,47 @@
         },
         {
           "id": 32000,
-          "logprob": -11.9140625,
+          "logprob": -11.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.34375,
+          "logprob": -11.3359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.578125,
+          "logprob": -11.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.953125,
+          "logprob": -11.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.296875,
+          "logprob": -11.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.875,
           "text": "<image>"
         },
         {
@@ -51639,27 +36865,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9765625,
+          "logprob": -12.046875,
           "text": "<image>"
         },
         {
@@ -51669,7 +36875,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.734375,
+          "logprob": -13.7265625,
           "text": "<image>"
         },
         {
@@ -51679,7 +36885,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.8359375,
+          "logprob": -13.8203125,
           "text": "<image>"
         },
         {
@@ -51694,37 +36900,37 @@
         },
         {
           "id": 32000,
-          "logprob": -14.5625,
+          "logprob": -14.546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.0234375,
+          "logprob": -14.015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5703125,
+          "logprob": -12.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8125,
+          "logprob": -11.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.7109375,
+          "logprob": -10.6953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6015625,
+          "logprob": -11.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8671875,
+          "logprob": -12.0234375,
           "text": "<image>"
         },
         {
@@ -51739,32 +36945,32 @@
         },
         {
           "id": 32000,
-          "logprob": -14.3359375,
+          "logprob": -14.34375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.90625,
+          "logprob": -11.8984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.296875,
+          "logprob": -13.4375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.984375,
+          "logprob": -11.9765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0625,
+          "logprob": -11.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4296875,
+          "logprob": -11.421875,
           "text": "<image>"
         },
         {
@@ -51779,27 +36985,27 @@
         },
         {
           "id": 32000,
-          "logprob": -11.28125,
+          "logprob": -11.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5546875,
+          "logprob": -11.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.671875,
+          "logprob": -11.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1171875,
+          "logprob": -12.125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5,
+          "logprob": -11.484375,
           "text": "<image>"
         },
         {
@@ -51809,27 +37015,27 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8984375,
+          "logprob": -11.875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2890625,
+          "logprob": -12.3125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6484375,
+          "logprob": -11.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8984375,
+          "logprob": -12.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.734375,
+          "logprob": -11.7421875,
           "text": "<image>"
         },
         {
@@ -51839,7 +37045,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.40625,
+          "logprob": -11.3984375,
           "text": "<image>"
         },
         {
@@ -51849,7 +37055,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.3515625,
+          "logprob": -11.34375,
           "text": "<image>"
         },
         {
@@ -51864,37 +37070,37 @@
         },
         {
           "id": 32000,
-          "logprob": -11.03125,
+          "logprob": -10.9609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.375,
+          "logprob": -13.265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6640625,
+          "logprob": -12.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.96875,
+          "logprob": -11.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.671875,
+          "logprob": -14.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.390625,
+          "logprob": -12.3828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.2890625,
+          "logprob": -14.3515625,
           "text": "<image>"
         },
         {
@@ -51904,32 +37110,32 @@
         },
         {
           "id": 32000,
-          "logprob": -15.1796875,
+          "logprob": -15.125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6875,
+          "logprob": -12.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7421875,
+          "logprob": -12.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.6484375,
+          "logprob": -14.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.546875,
+          "logprob": -13.5234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5703125,
+          "logprob": -12.5859375,
           "text": "<image>"
         },
         {
@@ -51939,32 +37145,17 @@
         },
         {
           "id": 32000,
-          "logprob": -14.390625,
+          "logprob": -14.3828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.96875,
+          "logprob": -12.9453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7421875,
+          "logprob": -12.609375,
           "text": "<image>"
         },
         {
@@ -51974,32 +37165,47 @@
         },
         {
           "id": 32000,
-          "logprob": -12.453125,
+          "logprob": -12.953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.8046875,
+          "logprob": -11.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.0078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.7890625,
+          "logprob": -12.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.3828125,
+          "logprob": -13.8828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.828125,
+          "logprob": -12.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8359375,
           "text": "<image>"
         },
         {
@@ -52009,12 +37215,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.0,
+          "logprob": -13.0078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1171875,
+          "logprob": -12.140625,
           "text": "<image>"
         },
         {
@@ -52024,17 +37230,17 @@
         },
         {
           "id": 32000,
-          "logprob": -14.015625,
+          "logprob": -13.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6328125,
+          "logprob": -11.625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4765625,
+          "logprob": -11.484375,
           "text": "<image>"
         },
         {
@@ -52044,7 +37250,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.5859375,
+          "logprob": -11.5703125,
           "text": "<image>"
         },
         {
@@ -52059,7 +37265,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0390625,
+          "logprob": -12.03125,
           "text": "<image>"
         },
         {
@@ -52069,7 +37275,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.4609375,
+          "logprob": -11.46875,
           "text": "<image>"
         },
         {
@@ -52079,7 +37285,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0625,
+          "logprob": -12.078125,
           "text": "<image>"
         },
         {
@@ -52089,142 +37295,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.53125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
+          "logprob": -11.296875,
           "text": "<image>"
         },
         {
@@ -52234,7 +37305,142 @@
         },
         {
           "id": 32000,
-          "logprob": -11.34375,
+          "logprob": -11.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.21875,
           "text": "<image>"
         },
         {
@@ -52244,52 +37450,52 @@
         },
         {
           "id": 32000,
-          "logprob": -11.3671875,
+          "logprob": -11.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0,
+          "logprob": -12.984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.1875,
+          "logprob": -15.15625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0,
+          "logprob": -11.9765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.75,
+          "logprob": -12.765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.515625,
+          "logprob": -13.4921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5703125,
+          "logprob": -11.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.703125,
+          "logprob": -11.6953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3671875,
+          "logprob": -11.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4921875,
+          "logprob": -11.5,
           "text": "<image>"
         },
         {
@@ -52299,12 +37505,12 @@
         },
         {
           "id": 32000,
-          "logprob": -14.5546875,
+          "logprob": -14.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.390625,
+          "logprob": -13.4140625,
           "text": "<image>"
         },
         {
@@ -52314,7 +37520,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.7265625,
+          "logprob": -11.71875,
           "text": "<image>"
         },
         {
@@ -52329,7 +37535,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.2421875,
+          "logprob": -13.2578125,
           "text": "<image>"
         },
         {
@@ -52349,27 +37555,27 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8203125,
+          "logprob": -11.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9296875,
+          "logprob": -11.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.796875,
+          "logprob": -11.765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1875,
+          "logprob": -11.1953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.359375,
+          "logprob": -13.3359375,
           "text": "<image>"
         },
         {
@@ -52379,7 +37585,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.6796875,
+          "logprob": -12.703125,
           "text": "<image>"
         },
         {
@@ -52394,27 +37600,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.5625,
+          "logprob": -13.6171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6484375,
+          "logprob": -13.328125,
           "text": "<image>"
         },
         {
@@ -52422,6 +37613,21 @@
           "logprob": -12.828125,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -13.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.796875,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -12.78125,
@@ -52429,42 +37635,42 @@
         },
         {
           "id": 32000,
-          "logprob": -12.578125,
+          "logprob": -12.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7578125,
+          "logprob": -12.765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.484375,
+          "logprob": -11.46875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5390625,
+          "logprob": -12.484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6015625,
+          "logprob": -11.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.640625,
+          "logprob": -11.6171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.140625,
+          "logprob": -12.1328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.65625,
+          "logprob": -12.59375,
           "text": "<image>"
         },
         {
@@ -52474,17 +37680,17 @@
         },
         {
           "id": 32000,
-          "logprob": -14.265625,
+          "logprob": -14.25,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0078125,
+          "logprob": -12.0234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7578125,
+          "logprob": -11.7734375,
           "text": "<image>"
         },
         {
@@ -52499,52 +37705,52 @@
         },
         {
           "id": 32000,
-          "logprob": -11.625,
+          "logprob": -11.609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.53125,
+          "logprob": -13.5234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.046875,
+          "logprob": -12.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6796875,
+          "logprob": -12.6953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7109375,
+          "logprob": -11.703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.890625,
+          "logprob": -11.8828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8359375,
+          "logprob": -11.84375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7734375,
+          "logprob": -11.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3671875,
+          "logprob": -12.3515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.3125,
+          "logprob": -14.3046875,
           "text": "<image>"
         },
         {
@@ -52554,22 +37760,22 @@
         },
         {
           "id": 32000,
-          "logprob": -11.921875,
+          "logprob": -11.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3203125,
+          "logprob": -11.328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1640625,
+          "logprob": -14.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.2421875,
+          "logprob": -11.25,
           "text": "<image>"
         },
         {
@@ -52579,7 +37785,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.078125,
           "text": "<image>"
         },
         {
@@ -52589,12 +37795,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.2578125,
+          "logprob": -12.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.7265625,
+          "logprob": -13.71875,
           "text": "<image>"
         },
         {
@@ -52614,12 +37820,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.3828125,
+          "logprob": -13.390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5390625,
+          "logprob": -12.515625,
           "text": "<image>"
         },
         {
@@ -52629,22 +37835,22 @@
         },
         {
           "id": 32000,
-          "logprob": -14.15625,
+          "logprob": -14.1796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5703125,
+          "logprob": -11.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5546875,
+          "logprob": -11.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.65625,
+          "logprob": -11.671875,
           "text": "<image>"
         },
         {
@@ -52654,47 +37860,47 @@
         },
         {
           "id": 32000,
-          "logprob": -12.6484375,
+          "logprob": -12.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5390625,
+          "logprob": -12.515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7734375,
+          "logprob": -11.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0859375,
+          "logprob": -13.1328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9140625,
+          "logprob": -12.8984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.6015625,
+          "logprob": -14.609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5390625,
+          "logprob": -11.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8828125,
+          "logprob": -11.8515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.078125,
+          "logprob": -13.0625,
           "text": "<image>"
         },
         {
@@ -52704,17 +37910,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.15625,
+          "logprob": -12.1328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.03125,
+          "logprob": -12.0234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4453125,
+          "logprob": -12.453125,
           "text": "<image>"
         },
         {
@@ -52724,7 +37930,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.9375,
+          "logprob": -11.9296875,
           "text": "<image>"
         },
         {
@@ -52739,42 +37945,42 @@
         },
         {
           "id": 32000,
-          "logprob": -12.7890625,
+          "logprob": -12.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0390625,
+          "logprob": -13.0234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8203125,
+          "logprob": -12.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.578125,
+          "logprob": -12.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.7578125,
+          "logprob": -10.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9609375,
+          "logprob": -11.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.484375,
+          "logprob": -11.4921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.265625,
+          "logprob": -11.25,
           "text": "<image>"
         },
         {
@@ -52784,17 +37990,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.71875,
+          "logprob": -12.6953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7890625,
+          "logprob": -11.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5234375,
+          "logprob": -11.515625,
           "text": "<image>"
         },
         {
@@ -52804,7 +38010,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.46875,
+          "logprob": -12.4765625,
           "text": "<image>"
         },
         {
@@ -52819,17 +38025,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.65625,
+          "logprob": -11.6484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5234375,
+          "logprob": -11.515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7890625,
+          "logprob": -12.8125,
           "text": "<image>"
         },
         {
@@ -52839,17 +38045,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.28125,
+          "logprob": -13.2890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.109375,
+          "logprob": -11.09375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.34375,
+          "logprob": -12.3359375,
           "text": "<image>"
         },
         {
@@ -52859,27 +38065,27 @@
         },
         {
           "id": 32000,
-          "logprob": -13.0,
+          "logprob": -13.0078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.28125,
+          "logprob": -15.328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.8046875,
+          "logprob": -14.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0234375,
+          "logprob": -11.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.109375,
+          "logprob": -13.0703125,
           "text": "<image>"
         },
         {
@@ -52889,12 +38095,12 @@
         },
         {
           "id": 32000,
-          "logprob": -14.0078125,
+          "logprob": -14.0859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.7109375,
+          "logprob": -14.734375,
           "text": "<image>"
         },
         {
@@ -52909,127 +38115,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.9765625,
+          "logprob": -13.953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.3046875,
+          "logprob": -14.28125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.78125,
+          "logprob": -14.8828125,
           "text": "<image>"
         },
         {
@@ -53039,22 +38135,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.859375,
+          "logprob": -12.28125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.765625,
+          "logprob": -12.34375,
           "text": "<image>"
         },
         {
@@ -53069,117 +38155,87 @@
         },
         {
           "id": 32000,
-          "logprob": -11.71875,
+          "logprob": -12.4296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.25,
+          "logprob": -11.8984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.65625,
+          "logprob": -11.984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.84375,
+          "logprob": -11.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0390625,
+          "logprob": -11.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4921875,
+          "logprob": -12.1875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8828125,
+          "logprob": -11.6171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1015625,
+          "logprob": -12.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.0625,
+          "logprob": -12.4765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.09375,
+          "logprob": -12.953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2578125,
+          "logprob": -10.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.4453125,
+          "logprob": -13.1015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9921875,
+          "logprob": -12.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.140625,
+          "logprob": -11.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.828125,
+          "logprob": -11.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5546875,
+          "logprob": -11.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6015625,
+          "logprob": -11.765625,
           "text": "<image>"
         },
         {
@@ -53189,32 +38245,182 @@
         },
         {
           "id": 32000,
-          "logprob": -12.84375,
+          "logprob": -12.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.78125,
+          "logprob": -11.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1171875,
+          "logprob": -12.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8046875,
+          "logprob": -11.71875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5234375,
+          "logprob": -11.34375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4609375,
+          "logprob": -12.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4765625,
           "text": "<image>"
         },
         {
@@ -53234,22 +38440,22 @@
         },
         {
           "id": 32000,
-          "logprob": -12.953125,
+          "logprob": -12.9609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.859375,
+          "logprob": -12.8515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0390625,
+          "logprob": -12.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.28125,
+          "logprob": -13.234375,
           "text": "<image>"
         },
         {
@@ -53259,12 +38465,12 @@
         },
         {
           "id": 32000,
-          "logprob": -10.8203125,
+          "logprob": -10.8515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.6953125,
+          "logprob": -10.671875,
           "text": "<image>"
         },
         {
@@ -53274,7 +38480,7 @@
         },
         {
           "id": 32000,
-          "logprob": -10.9375,
+          "logprob": -10.9140625,
           "text": "<image>"
         },
         {
@@ -53294,17 +38500,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.3046875,
+          "logprob": -12.2890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9609375,
+          "logprob": -10.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6328125,
+          "logprob": -11.6015625,
           "text": "<image>"
         },
         {
@@ -53314,17 +38520,17 @@
         },
         {
           "id": 32000,
-          "logprob": -10.84375,
+          "logprob": -10.8515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.140625,
+          "logprob": -11.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.09375,
+          "logprob": -11.0859375,
           "text": "<image>"
         },
         {
@@ -53334,12 +38540,12 @@
         },
         {
           "id": 32000,
-          "logprob": -15.578125,
+          "logprob": -15.765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.875,
+          "logprob": -11.8515625,
           "text": "<image>"
         },
         {
@@ -53354,22 +38560,22 @@
         },
         {
           "id": 32000,
-          "logprob": -10.984375,
+          "logprob": -10.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.234375,
+          "logprob": -12.25,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8359375,
+          "logprob": -11.828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8515625,
+          "logprob": -11.84375,
           "text": "<image>"
         },
         {
@@ -53384,7 +38590,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.5625,
+          "logprob": -13.5703125,
           "text": "<image>"
         },
         {
@@ -53394,37 +38600,37 @@
         },
         {
           "id": 32000,
-          "logprob": -13.5859375,
+          "logprob": -13.6171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.0234375,
+          "logprob": -14.0078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1796875,
+          "logprob": -11.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1484375,
+          "logprob": -14.140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.234375,
+          "logprob": -12.25,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.734375,
+          "logprob": -11.7265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3515625,
+          "logprob": -12.3203125,
           "text": "<image>"
         },
         {
@@ -53434,17 +38640,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.90625,
+          "logprob": -11.8828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3359375,
+          "logprob": -12.3125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5703125,
+          "logprob": -11.5625,
           "text": "<image>"
         },
         {
@@ -53454,17 +38660,17 @@
         },
         {
           "id": 32000,
-          "logprob": -14.6875,
+          "logprob": -14.6796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0625,
+          "logprob": -11.046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5234375,
+          "logprob": -11.5,
           "text": "<image>"
         },
         {
@@ -53489,12 +38695,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.46875,
+          "logprob": -12.40625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.0859375,
           "text": "<image>"
         },
         {
@@ -53507,21 +38713,6 @@
           "logprob": -12.46875,
           "text": "<image>"
         },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6640625,
-          "text": "<image>"
-        },
         {
           "id": 32000,
           "logprob": -11.9765625,
@@ -53529,7 +38720,22 @@
         },
         {
           "id": 32000,
-          "logprob": -10.8671875,
+          "logprob": -11.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.875,
           "text": "<image>"
         },
         {
@@ -53549,12 +38755,12 @@
         },
         {
           "id": 32000,
-          "logprob": -10.796875,
+          "logprob": -10.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.59375,
+          "logprob": -11.6171875,
           "text": "<image>"
         },
         {
@@ -53584,22 +38790,22 @@
         },
         {
           "id": 32000,
-          "logprob": -11.390625,
+          "logprob": -11.3828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9375,
+          "logprob": -10.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.6796875,
+          "logprob": -10.703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9375,
+          "logprob": -11.90625,
           "text": "<image>"
         },
         {
@@ -53609,82 +38815,57 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8125,
+          "logprob": -11.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3671875,
+          "logprob": -11.4453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -9.625,
+          "logprob": -9.6328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.8515625,
+          "logprob": -10.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9453125,
+          "logprob": -13.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8203125,
+          "logprob": -12.890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3359375,
+          "logprob": -13.4765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.015625,
+          "logprob": -14.109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8671875,
+          "logprob": -11.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4453125,
+          "logprob": -13.484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.75,
+          "logprob": -13.5625,
           "text": "<image>"
         },
         {
@@ -53692,6 +38873,31 @@
           "logprob": -12.859375,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -10.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.890625,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -13.0625,
@@ -53699,7 +38905,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.21875,
+          "logprob": -11.2265625,
           "text": "<image>"
         },
         {
@@ -53709,7 +38915,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.296875,
+          "logprob": -12.2890625,
           "text": "<image>"
         },
         {
@@ -53719,12 +38925,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.8671875,
+          "logprob": -12.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.6953125,
+          "logprob": -15.671875,
           "text": "<image>"
         },
         {
@@ -53739,7 +38945,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.140625,
+          "logprob": -12.1328125,
           "text": "<image>"
         },
         {
@@ -53749,22 +38955,22 @@
         },
         {
           "id": 32000,
-          "logprob": -11.390625,
+          "logprob": -11.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.328125,
+          "logprob": -12.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.265625,
+          "logprob": -12.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9453125,
+          "logprob": -10.9296875,
           "text": "<image>"
         },
         {
@@ -53779,7 +38985,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.6015625,
+          "logprob": -11.59375,
           "text": "<image>"
         },
         {
@@ -53789,7 +38995,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.546875,
+          "logprob": -11.53125,
           "text": "<image>"
         },
         {
@@ -53804,7 +39010,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.6796875,
+          "logprob": -11.6875,
           "text": "<image>"
         },
         {
@@ -53819,27 +39025,27 @@
         },
         {
           "id": 32000,
-          "logprob": -12.125,
+          "logprob": -12.1171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5703125,
+          "logprob": -12.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.109375,
+          "logprob": -12.1015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0078125,
+          "logprob": -12.0,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.734375,
+          "logprob": -13.7265625,
           "text": "<image>"
         },
         {
@@ -53849,67 +39055,67 @@
         },
         {
           "id": 32000,
-          "logprob": -11.6484375,
+          "logprob": -11.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.34375,
+          "logprob": -12.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.140625,
+          "logprob": -12.1484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1796875,
+          "logprob": -14.21875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.8046875,
+          "logprob": -14.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -18.046875,
+          "logprob": -17.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.34375,
+          "logprob": -15.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.4453125,
+          "logprob": -14.4765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.625,
+          "logprob": -10.6015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0546875,
+          "logprob": -12.0703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.859375,
+          "logprob": -12.84375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2265625,
+          "logprob": -12.2421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2109375,
+          "logprob": -13.203125,
           "text": "<image>"
         },
         {
@@ -53919,37 +39125,37 @@
         },
         {
           "id": 32000,
-          "logprob": -10.9296875,
+          "logprob": -10.9375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9140625,
+          "logprob": -11.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.125,
+          "logprob": -12.1015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9609375,
+          "logprob": -10.9453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7734375,
+          "logprob": -11.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.828125,
+          "logprob": -11.8203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2890625,
+          "logprob": -12.296875,
           "text": "<image>"
         },
         {
@@ -53959,27 +39165,27 @@
         },
         {
           "id": 32000,
-          "logprob": -11.53125,
+          "logprob": -11.5390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0234375,
+          "logprob": -12.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.2421875,
+          "logprob": -15.2265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1171875,
+          "logprob": -12.125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1171875,
+          "logprob": -12.125,
           "text": "<image>"
         },
         {
@@ -53989,7 +39195,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.671875,
+          "logprob": -11.6796875,
           "text": "<image>"
         },
         {
@@ -54009,32 +39215,32 @@
         },
         {
           "id": 32000,
-          "logprob": -12.734375,
+          "logprob": -12.703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.609375,
+          "logprob": -11.6328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5234375,
+          "logprob": -11.515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5078125,
+          "logprob": -11.546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2109375,
+          "logprob": -12.21875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.421875,
+          "logprob": -11.4140625,
           "text": "<image>"
         },
         {
@@ -54044,137 +39250,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.96875,
+          "logprob": -11.9765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.46875,
+          "logprob": -12.9609375,
           "text": "<image>"
         },
         {
@@ -54182,6 +39263,131 @@
           "logprob": -12.578125,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -11.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5078125,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -11.6171875,
@@ -54189,32 +39395,32 @@
         },
         {
           "id": 32000,
-          "logprob": -13.8125,
+          "logprob": -13.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1640625,
+          "logprob": -11.203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.765625,
+          "logprob": -11.78125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.5,
+          "logprob": -16.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3046875,
+          "logprob": -12.296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1171875,
+          "logprob": -14.1484375,
           "text": "<image>"
         },
         {
@@ -54224,12 +39430,47 @@
         },
         {
           "id": 32000,
-          "logprob": -12.5,
+          "logprob": -12.4921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.6015625,
+          "logprob": -10.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1953125,
           "text": "<image>"
         },
         {
@@ -54239,52 +39480,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.625,
+          "logprob": -11.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3046875,
+          "logprob": -12.6484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1171875,
+          "logprob": -11.109375,
           "text": "<image>"
         },
         {
@@ -54294,7 +39500,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.921875,
+          "logprob": -13.90625,
           "text": "<image>"
         },
         {
@@ -54309,12 +39515,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.3671875,
+          "logprob": -13.375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.7109375,
+          "logprob": -14.703125,
           "text": "<image>"
         },
         {
@@ -54324,7 +39530,7 @@
         },
         {
           "id": 32000,
-          "logprob": -14.859375,
+          "logprob": -14.890625,
           "text": "<image>"
         },
         {
@@ -54339,37 +39545,37 @@
         },
         {
           "id": 32000,
-          "logprob": -11.9296875,
+          "logprob": -11.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3828125,
+          "logprob": -13.3984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.046875,
+          "logprob": -13.9453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1484375,
+          "logprob": -13.1875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.265625,
+          "logprob": -11.28125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.125,
+          "logprob": -15.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.046875,
+          "logprob": -15.0234375,
           "text": "<image>"
         },
         {
@@ -54384,92 +39590,22 @@
         },
         {
           "id": 32000,
-          "logprob": -14.09375,
+          "logprob": -14.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.890625,
+          "logprob": -13.8984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.3203125,
+          "logprob": -15.3125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9609375,
+          "logprob": -12.8125,
           "text": "<image>"
         },
         {
@@ -54479,22 +39615,92 @@
         },
         {
           "id": 32000,
-          "logprob": -12.3203125,
+          "logprob": -13.046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.59375,
+          "logprob": -10.625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.171875,
+          "logprob": -11.25,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5,
+          "logprob": -12.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4921875,
           "text": "<image>"
         },
         {
@@ -54524,17 +39730,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.78125,
+          "logprob": -12.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3125,
+          "logprob": -11.3046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1171875,
+          "logprob": -11.109375,
           "text": "<image>"
         },
         {
@@ -54544,27 +39750,27 @@
         },
         {
           "id": 32000,
-          "logprob": -13.0859375,
+          "logprob": -13.0234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.921875,
+          "logprob": -10.9296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.515625,
+          "logprob": -13.5078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4921875,
+          "logprob": -11.5,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8203125,
+          "logprob": -11.8359375,
           "text": "<image>"
         },
         {
@@ -54574,42 +39780,42 @@
         },
         {
           "id": 32000,
-          "logprob": -14.0078125,
+          "logprob": -14.0234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.21875,
+          "logprob": -15.2578125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -17.84375,
+          "logprob": -17.953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.046875,
+          "logprob": -12.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.296875,
+          "logprob": -12.28125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7109375,
+          "logprob": -11.6875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1015625,
+          "logprob": -12.1171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.515625,
+          "logprob": -11.5546875,
           "text": "<image>"
         },
         {
@@ -54619,17 +39825,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.640625,
+          "logprob": -12.6171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4609375,
+          "logprob": -13.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.84375,
+          "logprob": -14.828125,
           "text": "<image>"
         },
         {
@@ -54639,22 +39845,22 @@
         },
         {
           "id": 32000,
-          "logprob": -11.6171875,
+          "logprob": -11.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.765625,
+          "logprob": -12.7421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5390625,
+          "logprob": -13.484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4453125,
+          "logprob": -11.4609375,
           "text": "<image>"
         },
         {
@@ -54664,37 +39870,37 @@
         },
         {
           "id": 32000,
-          "logprob": -11.96875,
+          "logprob": -11.984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4765625,
+          "logprob": -12.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.890625,
+          "logprob": -15.9375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.953125,
+          "logprob": -11.984375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.40625,
+          "logprob": -11.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0,
+          "logprob": -12.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1875,
+          "logprob": -14.3046875,
           "text": "<image>"
         },
         {
@@ -54704,7 +39910,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.125,
+          "logprob": -12.1171875,
           "text": "<image>"
         },
         {
@@ -54714,22 +39920,22 @@
         },
         {
           "id": 32000,
-          "logprob": -13.140625,
+          "logprob": -13.1484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1484375,
+          "logprob": -12.1875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.375,
+          "logprob": -13.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.9453125,
+          "logprob": -13.9921875,
           "text": "<image>"
         },
         {
@@ -54739,12 +39945,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0859375,
+          "logprob": -12.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.734375,
+          "logprob": -10.7421875,
           "text": "<image>"
         },
         {
@@ -54754,12 +39960,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.65625,
+          "logprob": -12.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.953125,
+          "logprob": -11.9765625,
           "text": "<image>"
         },
         {
@@ -54769,12 +39975,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.5,
+          "logprob": -12.4765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8984375,
+          "logprob": -11.890625,
           "text": "<image>"
         },
         {
@@ -54794,22 +40000,22 @@
         },
         {
           "id": 32000,
-          "logprob": -12.4375,
+          "logprob": -12.4609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5703125,
+          "logprob": -12.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.828125,
+          "logprob": -12.8203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.375,
+          "logprob": -13.359375,
           "text": "<image>"
         },
         {
@@ -54829,7 +40035,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.2421875,
+          "logprob": -11.234375,
           "text": "<image>"
         },
         {
@@ -54839,27 +40045,27 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.328125,
+          "logprob": -15.3046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.890625,
+          "logprob": -12.8828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4453125,
+          "logprob": -13.453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8203125,
+          "logprob": -11.8125,
           "text": "<image>"
         },
         {
@@ -54869,17 +40075,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.96875,
+          "logprob": -12.9765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9921875,
+          "logprob": -11.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.421875,
+          "logprob": -13.4296875,
           "text": "<image>"
         },
         {
@@ -54889,12 +40095,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.265625,
+          "logprob": -13.28125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.265625,
+          "logprob": -13.2578125,
           "text": "<image>"
         },
         {
@@ -54909,12 +40115,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.15625,
+          "logprob": -12.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.375,
+          "logprob": -10.3828125,
           "text": "<image>"
         },
         {
@@ -54924,7 +40130,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.2109375,
+          "logprob": -13.203125,
           "text": "<image>"
         },
         {
@@ -54934,7 +40140,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.15625,
+          "logprob": -12.1328125,
           "text": "<image>"
         },
         {
@@ -54944,12 +40150,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.171875,
+          "logprob": -13.1796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6328125,
+          "logprob": -11.65625,
           "text": "<image>"
         },
         {
@@ -54959,12 +40165,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.90625,
+          "logprob": -11.9375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.765625,
+          "logprob": -11.7734375,
           "text": "<image>"
         },
         {
@@ -54979,7 +40185,7 @@
         },
         {
           "id": 32000,
-          "logprob": -10.8984375,
+          "logprob": -10.90625,
           "text": "<image>"
         },
         {
@@ -54989,17 +40195,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.40625,
+          "logprob": -12.3828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5390625,
+          "logprob": -11.5234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.59375,
+          "logprob": -16.578125,
           "text": "<image>"
         },
         {
@@ -55009,7 +40215,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.9140625,
+          "logprob": -11.9453125,
           "text": "<image>"
         },
         {
@@ -55024,22 +40230,22 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8984375,
+          "logprob": -11.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0703125,
+          "logprob": -13.0625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.03125,
+          "logprob": -12.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.921875,
+          "logprob": -11.9296875,
           "text": "<image>"
         },
         {
@@ -55049,12 +40255,12 @@
         },
         {
           "id": 32000,
-          "logprob": -10.5390625,
+          "logprob": -10.53125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.453125,
+          "logprob": -11.4375,
           "text": "<image>"
         },
         {
@@ -55064,22 +40270,22 @@
         },
         {
           "id": 32000,
-          "logprob": -13.0859375,
+          "logprob": -13.09375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.1796875,
+          "logprob": -12.15625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.8203125,
+          "logprob": -15.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.984375,
+          "logprob": -10.9765625,
           "text": "<image>"
         },
         {
@@ -55089,42 +40295,42 @@
         },
         {
           "id": 32000,
-          "logprob": -10.109375,
+          "logprob": -10.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8671875,
+          "logprob": -11.859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1328125,
+          "logprob": -13.125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.640625,
+          "logprob": -12.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.015625,
+          "logprob": -16.046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5078125,
+          "logprob": -11.515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.7265625,
+          "logprob": -13.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.703125,
+          "logprob": -14.671875,
           "text": "<image>"
         },
         {
@@ -55134,32 +40340,32 @@
         },
         {
           "id": 32000,
-          "logprob": -12.2421875,
+          "logprob": -12.2109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.6640625,
+          "logprob": -10.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.6640625,
+          "logprob": -15.6484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9140625,
+          "logprob": -10.9296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3203125,
+          "logprob": -11.328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.046875,
+          "logprob": -11.0546875,
           "text": "<image>"
         },
         {
@@ -55169,7 +40375,7 @@
         },
         {
           "id": 32000,
-          "logprob": -10.953125,
+          "logprob": -10.96875,
           "text": "<image>"
         },
         {
@@ -55184,337 +40390,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2578125,
+          "logprob": -12.359375,
           "text": "<image>"
         },
         {
@@ -55524,22 +40400,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.84375,
+          "logprob": -14.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
+          "logprob": -11.4921875,
           "text": "<image>"
         },
         {
@@ -55549,87 +40415,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.125,
+          "logprob": -12.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.25,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.390625,
+          "logprob": -12.640625,
           "text": "<image>"
         },
         {
@@ -55639,87 +40430,297 @@
         },
         {
           "id": 32000,
-          "logprob": -12.5078125,
+          "logprob": -12.4375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.09375,
+          "logprob": -11.8828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.8515625,
+          "logprob": -12.15625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8203125,
+          "logprob": -11.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.734375,
+          "logprob": -13.0,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2578125,
+          "logprob": -12.0703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.28125,
+          "logprob": -11.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.859375,
+          "logprob": -13.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.703125,
+          "logprob": -12.1484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5703125,
+          "logprob": -13.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.59375,
+          "logprob": -12.203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.890625,
+          "logprob": -11.9375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.3359375,
+          "logprob": -12.1796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0625,
+          "logprob": -10.7421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.53125,
+          "logprob": -10.8203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7109375,
+          "logprob": -12.375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.7578125,
+          "logprob": -11.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.25,
           "text": "<image>"
         },
         {
@@ -55729,17 +40730,47 @@
         },
         {
           "id": 32000,
-          "logprob": -10.75,
+          "logprob": -11.8515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.2265625,
+          "logprob": -13.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.9296875,
+          "logprob": -10.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1328125,
           "text": "<image>"
         },
         {
@@ -55749,7 +40780,12 @@
         },
         {
           "id": 32000,
-          "logprob": -10.5390625,
+          "logprob": -14.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0546875,
           "text": "<image>"
         },
         {
@@ -55757,6 +40793,56 @@
           "logprob": -11.6796875,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -11.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.234375,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -12.5390625,
@@ -55764,7 +40850,127 @@
         },
         {
           "id": 32000,
-          "logprob": -16.953125,
+          "logprob": -15.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.0,
           "text": "<image>"
         },
         {
@@ -55774,12 +40980,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.3359375,
+          "logprob": -11.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.671875,
+          "logprob": -11.6796875,
           "text": "<image>"
         },
         {
@@ -55794,12 +41000,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8671875,
+          "logprob": -11.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.484375,
+          "logprob": -11.4765625,
           "text": "<image>"
         },
         {
@@ -55809,12 +41015,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.6640625,
+          "logprob": -12.6875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4296875,
+          "logprob": -13.421875,
           "text": "<image>"
         },
         {
@@ -55829,7 +41035,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.3046875,
+          "logprob": -12.3125,
           "text": "<image>"
         },
         {
@@ -55839,17 +41045,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.34375,
+          "logprob": -13.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.0703125,
+          "logprob": -13.0625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.234375,
+          "logprob": -11.2421875,
           "text": "<image>"
         },
         {
@@ -55859,37 +41065,37 @@
         },
         {
           "id": 32000,
-          "logprob": -13.4921875,
+          "logprob": -13.5,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.7578125,
+          "logprob": -10.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.53125,
+          "logprob": -13.5390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6953125,
+          "logprob": -12.703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.203125,
+          "logprob": -13.1953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -17.078125,
+          "logprob": -17.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.25,
+          "logprob": -15.0234375,
           "text": "<image>"
         },
         {
@@ -55899,17 +41105,17 @@
         },
         {
           "id": 32000,
-          "logprob": -12.140625,
+          "logprob": -12.125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5234375,
+          "logprob": -11.515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.0625,
+          "logprob": -16.125,
           "text": "<image>"
         },
         {
@@ -55919,17 +41125,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.6015625,
+          "logprob": -11.609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.5859375,
+          "logprob": -14.6015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.890625,
+          "logprob": -11.8984375,
           "text": "<image>"
         },
         {
@@ -55944,22 +41150,22 @@
         },
         {
           "id": 32000,
-          "logprob": -11.7734375,
+          "logprob": -11.78125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9921875,
+          "logprob": -13.0,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9375,
+          "logprob": -12.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9921875,
+          "logprob": -12.0,
           "text": "<image>"
         },
         {
@@ -55969,7 +41175,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.7578125,
+          "logprob": -11.7734375,
           "text": "<image>"
         },
         {
@@ -55979,7 +41185,7 @@
         },
         {
           "id": 32000,
-          "logprob": -14.328125,
+          "logprob": -14.3203125,
           "text": "<image>"
         },
         {
@@ -55989,22 +41195,22 @@
         },
         {
           "id": 32000,
-          "logprob": -13.84375,
+          "logprob": -13.875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.2109375,
+          "logprob": -14.3203125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.390625,
+          "logprob": -12.375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.953125,
+          "logprob": -13.9375,
           "text": "<image>"
         },
         {
@@ -56014,27 +41220,27 @@
         },
         {
           "id": 32000,
-          "logprob": -14.6796875,
+          "logprob": -14.609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.84375,
+          "logprob": -11.8359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.65625,
+          "logprob": -10.6484375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0078125,
+          "logprob": -12.0,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0859375,
+          "logprob": -11.0703125,
           "text": "<image>"
         },
         {
@@ -56054,7 +41260,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.6875,
+          "logprob": -12.6640625,
           "text": "<image>"
         },
         {
@@ -56069,22 +41275,22 @@
         },
         {
           "id": 32000,
-          "logprob": -16.109375,
+          "logprob": -16.09375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.984375,
+          "logprob": -11.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.40625,
+          "logprob": -13.4140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.375,
+          "logprob": -13.453125,
           "text": "<image>"
         },
         {
@@ -56094,57 +41300,57 @@
         },
         {
           "id": 32000,
-          "logprob": -12.921875,
+          "logprob": -12.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.859375,
+          "logprob": -14.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3671875,
+          "logprob": -13.375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0546875,
+          "logprob": -12.0703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.21875,
+          "logprob": -15.234375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6953125,
+          "logprob": -11.6875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -10.5234375,
+          "logprob": -10.5390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.8984375,
+          "logprob": -14.703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.8515625,
+          "logprob": -13.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.6796875,
+          "logprob": -11.6875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -17.4375,
+          "logprob": -17.453125,
           "text": "<image>"
         },
         {
@@ -56159,7 +41365,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8125,
+          "logprob": -11.7890625,
           "text": "<image>"
         },
         {
@@ -56174,12 +41380,12 @@
         },
         {
           "id": 32000,
-          "logprob": -14.2421875,
+          "logprob": -14.265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.7265625,
+          "logprob": -13.6953125,
           "text": "<image>"
         },
         {
@@ -56194,57 +41400,57 @@
         },
         {
           "id": 32000,
-          "logprob": -12.84375,
+          "logprob": -12.890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9140625,
+          "logprob": -11.9375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.7421875,
+          "logprob": -14.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.984375,
+          "logprob": -11.9921875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.71875,
+          "logprob": -14.7265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.578125,
+          "logprob": -11.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.015625,
+          "logprob": -13.0,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.796875,
+          "logprob": -12.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0078125,
+          "logprob": -11.015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.46875,
+          "logprob": -16.421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9296875,
+          "logprob": -11.921875,
           "text": "<image>"
         },
         {
@@ -56254,7 +41460,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.8515625,
+          "logprob": -12.859375,
           "text": "<image>"
         },
         {
@@ -56269,12 +41475,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.140625,
+          "logprob": -12.1328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.359375,
+          "logprob": -13.3515625,
           "text": "<image>"
         },
         {
@@ -56289,27 +41495,27 @@
         },
         {
           "id": 32000,
-          "logprob": -11.578125,
+          "logprob": -11.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.8203125,
+          "logprob": -12.84375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.984375,
+          "logprob": -11.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5078125,
+          "logprob": -12.515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.625,
+          "logprob": -12.6328125,
           "text": "<image>"
         },
         {
@@ -56324,7 +41530,7 @@
         },
         {
           "id": 32000,
-          "logprob": -14.0546875,
+          "logprob": -14.046875,
           "text": "<image>"
         },
         {
@@ -56334,7 +41540,7 @@
         },
         {
           "id": 32000,
-          "logprob": -17.21875,
+          "logprob": -17.203125,
           "text": "<image>"
         },
         {
@@ -56344,17 +41550,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.3671875,
+          "logprob": -13.375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.6171875,
+          "logprob": -14.609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9609375,
+          "logprob": -12.953125,
           "text": "<image>"
         },
         {
@@ -56364,12 +41570,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.7265625,
+          "logprob": -11.71875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.03125,
+          "logprob": -15.0234375,
           "text": "<image>"
         },
         {
@@ -56379,7 +41585,7 @@
         },
         {
           "id": 32000,
-          "logprob": -15.890625,
+          "logprob": -15.9296875,
           "text": "<image>"
         },
         {
@@ -56404,22 +41610,22 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0234375,
+          "logprob": -12.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7578125,
+          "logprob": -12.75,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.7578125,
+          "logprob": -14.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5,
+          "logprob": -11.484375,
           "text": "<image>"
         },
         {
@@ -56429,7 +41635,7 @@
         },
         {
           "id": 32000,
-          "logprob": -10.7265625,
+          "logprob": -10.734375,
           "text": "<image>"
         },
         {
@@ -56439,22 +41645,22 @@
         },
         {
           "id": 32000,
-          "logprob": -11.4140625,
+          "logprob": -11.421875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9453125,
+          "logprob": -12.96875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1171875,
+          "logprob": -13.0625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0390625,
+          "logprob": -12.03125,
           "text": "<image>"
         },
         {
@@ -56464,7 +41670,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.234375,
+          "logprob": -12.2109375,
           "text": "<image>"
         },
         {
@@ -56479,7 +41685,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.0390625,
+          "logprob": -13.09375,
           "text": "<image>"
         },
         {
@@ -56489,7 +41695,7 @@
         },
         {
           "id": 32000,
-          "logprob": -15.265625,
+          "logprob": -15.2578125,
           "text": "<image>"
         },
         {
@@ -56504,7 +41710,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.453125,
+          "logprob": -11.4609375,
           "text": "<image>"
         },
         {
@@ -56519,12 +41725,12 @@
         },
         {
           "id": 32000,
-          "logprob": -10.6796875,
+          "logprob": -10.6875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.0546875,
           "text": "<image>"
         },
         {
@@ -56534,7 +41740,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.421875,
+          "logprob": -12.4140625,
           "text": "<image>"
         },
         {
@@ -56544,27 +41750,27 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8046875,
+          "logprob": -11.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.3828125,
+          "logprob": -14.4609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.734375,
+          "logprob": -11.7109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.890625,
+          "logprob": -12.828125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.859375,
+          "logprob": -12.8359375,
           "text": "<image>"
         },
         {
@@ -56574,27 +41780,27 @@
         },
         {
           "id": 32000,
-          "logprob": -13.5625,
+          "logprob": -13.5390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.25,
+          "logprob": -16.21875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.8125,
+          "logprob": -16.84375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.921875,
+          "logprob": -11.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.375,
+          "logprob": -14.359375,
           "text": "<image>"
         },
         {
@@ -56604,12 +41810,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.671875,
+          "logprob": -13.6953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.9921875,
+          "logprob": -12.984375,
           "text": "<image>"
         },
         {
@@ -56624,57 +41830,37 @@
         },
         {
           "id": 32000,
-          "logprob": -17.5625,
+          "logprob": -17.59375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.625,
+          "logprob": -12.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.0625,
+          "logprob": -15.046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.75,
+          "logprob": -16.71875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.84375,
+          "logprob": -11.8359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.328125,
+          "logprob": -13.3515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
+          "logprob": -12.2734375,
           "text": "<image>"
         },
         {
@@ -56684,17 +41870,37 @@
         },
         {
           "id": 32000,
-          "logprob": -13.109375,
+          "logprob": -11.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.625,
+          "logprob": -12.46875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.890625,
+          "logprob": -11.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.921875,
           "text": "<image>"
         },
         {
@@ -56704,7 +41910,7 @@
         },
         {
           "id": 32000,
-          "logprob": -13.15625,
+          "logprob": -13.21875,
           "text": "<image>"
         },
         {
@@ -56724,7 +41930,7 @@
         },
         {
           "id": 32000,
-          "logprob": -12.5390625,
+          "logprob": -12.5625,
           "text": "<image>"
         },
         {
@@ -56734,12 +41940,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.9609375,
+          "logprob": -11.9453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.828125,
+          "logprob": -11.8359375,
           "text": "<image>"
         },
         {
@@ -56749,22 +41955,22 @@
         },
         {
           "id": 32000,
-          "logprob": -12.8125,
+          "logprob": -12.875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.96875,
+          "logprob": -11.9609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.109375,
+          "logprob": -13.078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6328125,
+          "logprob": -12.625,
           "text": "<image>"
         },
         {
@@ -56784,12 +41990,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.109375,
+          "logprob": -12.0859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.265625,
+          "logprob": -14.25,
           "text": "<image>"
         },
         {
@@ -56799,127 +42005,17 @@
         },
         {
           "id": 32000,
-          "logprob": -16.640625,
+          "logprob": -16.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.828125,
+          "logprob": -12.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -19.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.328125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5859375,
+          "logprob": -12.7890625,
           "text": "<image>"
         },
         {
@@ -56929,852 +42025,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.3046875,
+          "logprob": -12.3125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.21875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.46875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.7734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.4765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -17.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.8125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.8671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.84375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6015625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.40625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.640625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5234375,
+          "logprob": -12.3203125,
           "text": "<image>"
         },
         {
@@ -57784,107 +42040,137 @@
         },
         {
           "id": 32000,
-          "logprob": -11.6875,
+          "logprob": -14.2734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.765625,
+          "logprob": -14.90625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.4140625,
+          "logprob": -12.8125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -18.703125,
+          "logprob": -14.546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.6484375,
+          "logprob": -19.4375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0703125,
+          "logprob": -12.2109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.703125,
+          "logprob": -13.65625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.0859375,
+          "logprob": -11.8515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.0703125,
+          "logprob": -11.4453125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.5234375,
+          "logprob": -12.2265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.3671875,
+          "logprob": -12.6640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.0390625,
+          "logprob": -12.3359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.640625,
+          "logprob": -17.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8359375,
+          "logprob": -17.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.1015625,
+          "logprob": -11.515625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3515625,
+          "logprob": -12.0078125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.578125,
+          "logprob": -13.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.25,
+          "logprob": -12.5703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.15625,
+          "logprob": -12.0546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7265625,
+          "logprob": -12.328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.90625,
+          "logprob": -13.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.5,
           "text": "<image>"
         },
         {
@@ -57899,17 +42185,32 @@
         },
         {
           "id": 32000,
-          "logprob": -10.5625,
+          "logprob": -14.875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.171875,
+          "logprob": -11.03125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.2265625,
+          "logprob": -11.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8984375,
           "text": "<image>"
         },
         {
@@ -57919,137 +42220,37 @@
         },
         {
           "id": 32000,
-          "logprob": -11.7578125,
+          "logprob": -11.328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.875,
+          "logprob": -11.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4453125,
+          "logprob": -11.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1796875,
+          "logprob": -15.046875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9453125,
+          "logprob": -12.2109375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8203125,
+          "logprob": -12.0390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.2421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.6953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.3671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4765625,
+          "logprob": -16.875,
           "text": "<image>"
         },
         {
@@ -58059,22 +42260,227 @@
         },
         {
           "id": 32000,
-          "logprob": -15.484375,
+          "logprob": -14.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.6328125,
+          "logprob": -11.9609375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.6015625,
+          "logprob": -12.796875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.75,
+          "logprob": -11.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.890625,
           "text": "<image>"
         },
         {
@@ -58084,7 +42490,807 @@
         },
         {
           "id": 32000,
-          "logprob": -13.3125,
+          "logprob": -13.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -18.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.0859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2578125,
           "text": "<image>"
         },
         {
@@ -58094,12 +43300,12 @@
         },
         {
           "id": 32000,
-          "logprob": -13.484375,
+          "logprob": -13.5390625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.2734375,
+          "logprob": -13.265625,
           "text": "<image>"
         },
         {
@@ -58114,17 +43320,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.6796875,
+          "logprob": -11.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.53125,
+          "logprob": -16.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.1484375,
+          "logprob": -14.171875,
           "text": "<image>"
         },
         {
@@ -58134,17 +43340,17 @@
         },
         {
           "id": 32000,
-          "logprob": -11.8125,
+          "logprob": -11.7890625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.03125,
+          "logprob": -15.1171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.484375,
+          "logprob": -15.4453125,
           "text": "<image>"
         },
         {
@@ -58159,37 +43365,37 @@
         },
         {
           "id": 32000,
-          "logprob": -10.6953125,
+          "logprob": -10.703125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.1640625,
+          "logprob": -11.15625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.84375,
+          "logprob": -12.9140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5625,
+          "logprob": -12.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7421875,
+          "logprob": -12.734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -18.0,
+          "logprob": -17.953125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.5859375,
+          "logprob": -11.59375,
           "text": "<image>"
         },
         {
@@ -58204,12 +43410,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.6796875,
+          "logprob": -11.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.71875,
+          "logprob": -12.8515625,
           "text": "<image>"
         },
         {
@@ -58219,27 +43425,27 @@
         },
         {
           "id": 32000,
-          "logprob": -12.1875,
+          "logprob": -12.171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.7578125,
+          "logprob": -12.7265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.125,
+          "logprob": -15.140625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -14.9140625,
+          "logprob": -14.9375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -15.546875,
+          "logprob": -15.6875,
           "text": "<image>"
         },
         {
@@ -58249,22 +43455,22 @@
         },
         {
           "id": 32000,
-          "logprob": -14.109375,
+          "logprob": -14.125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.234375,
+          "logprob": -13.2265625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.15625,
+          "logprob": -13.1640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.734375,
+          "logprob": -12.703125,
           "text": "<image>"
         },
         {
@@ -58274,7 +43480,7 @@
         },
         {
           "id": 32000,
-          "logprob": -14.7734375,
+          "logprob": -14.765625,
           "text": "<image>"
         },
         {
@@ -58284,12 +43490,12 @@
         },
         {
           "id": 32000,
-          "logprob": -11.7421875,
+          "logprob": -11.71875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.921875,
+          "logprob": -11.9296875,
           "text": "<image>"
         },
         {
@@ -58299,17 +43505,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.6015625,
+          "logprob": -13.6328125,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4375,
+          "logprob": -11.4296875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5,
+          "logprob": -13.5078125,
           "text": "<image>"
         },
         {
@@ -58324,27 +43530,27 @@
         },
         {
           "id": 32000,
-          "logprob": -12.1328125,
+          "logprob": -12.15625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.90625,
+          "logprob": -13.1171875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.578125,
+          "logprob": -12.5625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.3359375,
+          "logprob": -13.359375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.7265625,
+          "logprob": -11.734375,
           "text": "<image>"
         },
         {
@@ -58352,491 +43558,11 @@
           "logprob": -10.953125,
           "text": "<image>"
         },
-        {
-          "id": 32000,
-          "logprob": -12.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.0859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -19.65625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.859375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.71875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.4921875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.28125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.96875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3046875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.5390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.1484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.1796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.796875,
-          "text": "<image>"
-        },
         {
           "id": 32000,
           "logprob": -12.75,
           "text": "<image>"
         },
-        {
-          "id": 32000,
-          "logprob": -14.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.2890625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.8828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.6171875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.75,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.5625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9453125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -10.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.2109375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.234375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.9765625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.4609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6484375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.296875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.5703125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3203125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.1953125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.7421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.09375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.0546875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.59375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.3515625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.90625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.609375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.671875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -15.2265625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.78125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.6875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0390625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.796875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.03125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -16.515625,
-          "text": "<image>"
-        },
         {
           "id": 32000,
           "logprob": -12.7734375,
@@ -58844,7 +43570,22 @@
         },
         {
           "id": 32000,
-          "logprob": -12.4609375,
+          "logprob": -13.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5390625,
           "text": "<image>"
         },
         {
@@ -58854,87 +43595,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.171875,
+          "logprob": -19.671875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.8828125,
+          "logprob": -12.4765625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.4375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.828125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.2734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.734375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.3984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.0078125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.578125,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.3359375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -11.984375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -13.421875,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.9140625,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -14.34375,
-          "text": "<image>"
-        },
-        {
-          "id": 32000,
-          "logprob": -12.8828125,
+          "logprob": -11.609375,
           "text": "<image>"
         },
         {
@@ -58947,6 +43618,541 @@
           "logprob": -13.3203125,
           "text": "<image>"
         },
+        {
+          "id": 32000,
+          "logprob": -12.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.359375,
+          "text": "<image>"
+        },
         {
           "id": 32000,
           "logprob": -13.2109375,
@@ -58954,7 +44160,7 @@
         },
         {
           "id": 32000,
-          "logprob": -11.9765625,
+          "logprob": -11.984375,
           "text": "<image>"
         },
         {
@@ -58964,12 +44170,12 @@
         },
         {
           "id": 32000,
-          "logprob": -12.0078125,
+          "logprob": -12.015625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.0390625,
+          "logprob": -11.03125,
           "text": "<image>"
         },
         {
@@ -58984,17 +44190,17 @@
         },
         {
           "id": 32000,
-          "logprob": -13.8203125,
+          "logprob": -13.7734375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -13.5078125,
+          "logprob": -13.5546875,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.734375,
+          "logprob": -11.7265625,
           "text": "<image>"
         },
         {
@@ -59004,22 +44210,22 @@
         },
         {
           "id": 32000,
-          "logprob": -17.3125,
+          "logprob": -16.84375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5234375,
+          "logprob": -12.5859375,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -17.625,
+          "logprob": -17.640625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -11.9296875,
+          "logprob": -11.9453125,
           "text": "<image>"
         },
         {
@@ -59029,17 +44235,17 @@
         },
         {
           "id": 32000,
-          "logprob": -15.9140625,
+          "logprob": -16.0,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -16.65625,
+          "logprob": -16.625,
           "text": "<image>"
         },
         {
           "id": 32000,
-          "logprob": -12.5,
+          "logprob": -12.4921875,
           "text": "<image>"
         },
         {
@@ -59049,17 +44255,17 @@
         },
         {
           "id": 368,
-          "logprob": -0.19726562,
+          "logprob": -0.19604492,
           "text": "you"
         },
         {
           "id": 1912,
-          "logprob": -1.4990234,
+          "logprob": -1.5058594,
           "text": "tell"
         },
         {
           "id": 528,
-          "logprob": -0.31152344,
+          "logprob": -0.31030273,
           "text": "me"
         },
         {
@@ -59074,22 +44280,22 @@
         },
         {
           "id": 2485,
-          "logprob": -0.9941406,
+          "logprob": -0.9975586,
           "text": "short"
         },
         {
           "id": 2838,
-          "logprob": -0.46118164,
+          "logprob": -0.4633789,
           "text": "story"
         },
         {
           "id": 2818,
-          "logprob": -3.3183594,
+          "logprob": -3.3144531,
           "text": "based"
         },
         {
           "id": 356,
-          "logprob": -0.029129028,
+          "logprob": -0.029037476,
           "text": "on"
         },
         {
@@ -59099,12 +44305,12 @@
         },
         {
           "id": 3469,
-          "logprob": -0.29052734,
+          "logprob": -0.2890625,
           "text": "image"
         },
         {
           "id": 28804,
-          "logprob": -0.43188477,
+          "logprob": -0.42895508,
           "text": "?"
         }
       ],
@@ -59112,13 +44318,14807 @@
       "tokens": [
         {
           "id": 13,
-          "logprob": -0.0076828003,
+          "logprob": -0.007621765,
           "special": false,
           "text": "\n"
         },
         {
           "id": 13,
-          "logprob": -0.19958496,
+          "logprob": -0.20275879,
+          "special": false,
+          "text": "\n"
+        },
+        {
+          "id": 16114,
+          "logprob": -1.2578125,
+          "special": false,
+          "text": "Once"
+        },
+        {
+          "id": 3714,
+          "logprob": -0.2084961,
+          "special": false,
+          "text": " upon"
+        },
+        {
+          "id": 264,
+          "logprob": -0.0017738342,
+          "special": false,
+          "text": " a"
+        },
+        {
+          "id": 727,
+          "logprob": -0.011932373,
+          "special": false,
+          "text": " time"
+        },
+        {
+          "id": 28725,
+          "logprob": -0.17297363,
+          "special": false,
+          "text": ","
+        },
+        {
+          "id": 736,
+          "logprob": -0.9057617,
+          "special": false,
+          "text": " there"
+        },
+        {
+          "id": 403,
+          "logprob": -0.05758667,
+          "special": false,
+          "text": " was"
+        },
+        {
+          "id": 264,
+          "logprob": -0.00970459,
+          "special": false,
+          "text": " a"
+        }
+      ],
+      "top_tokens": null
+    },
+    "generated_text": "\n\nOnce upon a time, there was a"
+  },
+  {
+    "details": {
+      "best_of_sequences": null,
+      "finish_reason": "length",
+      "generated_tokens": 10,
+      "prefill": [
+        {
+          "id": 1,
+          "logprob": null,
+          "text": "<s>"
+        },
+        {
+          "id": 1247,
+          "logprob": -2.390625,
+          "text": "User"
+        },
+        {
+          "id": 28747,
+          "logprob": -12.328125,
+          "text": ":"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.4921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.1015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.4609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -9.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -18.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.5234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.2265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -9.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.1484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.4609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.0859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.4609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.4609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.8984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -9.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -9.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.4296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.4921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -18.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.4921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.2265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.4296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.1484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.1484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.4296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.4921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -9.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.2265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.1484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.4296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.4921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -9.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.4609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.25,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -19.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.46875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.1953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.21875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -18.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.0859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.65625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.234375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.59375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.9375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.6875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.15625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.75,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.09375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -19.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.4765625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.53125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.1171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.8203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.7578125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.6796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3515625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -10.9296875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.203125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.1796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.953125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.90625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.9609375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.28125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.328125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.1875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7421875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.078125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.0625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.5703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.671875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.3359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.9140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.6171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.6640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.34375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.796875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.703125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7890625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.78125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.171875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.8828125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.2734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.0390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.6484375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.3046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.96875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.3984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.921875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.359375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.2109375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.984375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -15.140625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.015625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.03125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -14.40625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.046875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.7734375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.5546875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.7265625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -13.390625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.84375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.5859375,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -17.640625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -11.9453125,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.71875,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.0,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -16.625,
+          "text": "<image>"
+        },
+        {
+          "id": 32000,
+          "logprob": -12.4921875,
+          "text": "<image>"
+        },
+        {
+          "id": 2418,
+          "logprob": -19.0625,
+          "text": "Can"
+        },
+        {
+          "id": 368,
+          "logprob": -0.19604492,
+          "text": "you"
+        },
+        {
+          "id": 1912,
+          "logprob": -1.5058594,
+          "text": "tell"
+        },
+        {
+          "id": 528,
+          "logprob": -0.31030273,
+          "text": "me"
+        },
+        {
+          "id": 264,
+          "logprob": -2.6367188,
+          "text": "a"
+        },
+        {
+          "id": 1215,
+          "logprob": -9.1015625,
+          "text": "very"
+        },
+        {
+          "id": 2485,
+          "logprob": -0.9975586,
+          "text": "short"
+        },
+        {
+          "id": 2838,
+          "logprob": -0.4633789,
+          "text": "story"
+        },
+        {
+          "id": 2818,
+          "logprob": -3.3144531,
+          "text": "based"
+        },
+        {
+          "id": 356,
+          "logprob": -0.029037476,
+          "text": "on"
+        },
+        {
+          "id": 272,
+          "logprob": -0.9902344,
+          "text": "the"
+        },
+        {
+          "id": 3469,
+          "logprob": -0.2890625,
+          "text": "image"
+        },
+        {
+          "id": 28804,
+          "logprob": -0.42895508,
+          "text": "?"
+        }
+      ],
+      "seed": null,
+      "tokens": [
+        {
+          "id": 13,
+          "logprob": -0.007621765,
+          "special": false,
+          "text": "\n"
+        },
+        {
+          "id": 13,
+          "logprob": -0.20812988,
           "special": false,
           "text": "\n"
         },
@@ -59130,37 +59130,37 @@
         },
         {
           "id": 3714,
-          "logprob": -0.20861816,
+          "logprob": -0.20825195,
           "special": false,
           "text": " upon"
         },
         {
           "id": 264,
-          "logprob": -0.0017719269,
+          "logprob": -0.0017709732,
           "special": false,
           "text": " a"
         },
         {
           "id": 727,
-          "logprob": -0.011749268,
+          "logprob": -0.011932373,
           "special": false,
           "text": " time"
         },
         {
           "id": 28725,
-          "logprob": -0.17529297,
+          "logprob": -0.17297363,
           "special": false,
           "text": ","
         },
         {
           "id": 736,
-          "logprob": -0.9086914,
+          "logprob": -0.9057617,
           "special": false,
           "text": " there"
         },
         {
           "id": 403,
-          "logprob": -0.056732178,
+          "logprob": -0.05758667,
           "special": false,
           "text": " was"
         },
diff --git a/integration-tests/models/__snapshots__/test_mamba/test_mamba_all_params.json b/integration-tests/models/__snapshots__/test_mamba/test_mamba_all_params.json
index 93724fe4..ef88926c 100644
--- a/integration-tests/models/__snapshots__/test_mamba/test_mamba_all_params.json
+++ b/integration-tests/models/__snapshots__/test_mamba/test_mamba_all_params.json
@@ -11,22 +11,22 @@
       },
       {
         "id": 13,
-        "logprob": -2.59375,
+        "logprob": -2.734375,
         "text": ","
       },
       {
         "id": 8862,
-        "logprob": -3.5625,
+        "logprob": -3.6875,
         "text": " yellow"
       },
       {
         "id": 13,
-        "logprob": -0.44726562,
+        "logprob": -0.40234375,
         "text": ","
       },
       {
         "id": 209,
-        "logprob": -8.0,
+        "logprob": -8.25,
         "text": " "
       }
     ],
@@ -52,7 +52,7 @@
       },
       {
         "id": 9830,
-        "logprob": -2.03125,
+        "logprob": -2.25,
         "special": false,
         "text": " colors"
       },
@@ -64,13 +64,13 @@
       },
       {
         "id": 329,
-        "logprob": -2.734375,
+        "logprob": -2.171875,
         "special": false,
         "text": " A"
       },
       {
         "id": 1180,
-        "logprob": -2.0,
+        "logprob": -2.046875,
         "special": false,
         "text": " number"
       },
@@ -81,19 +81,19 @@
         "text": " of"
       },
       {
-        "id": 253,
-        "logprob": -0.69140625,
+        "id": 1027,
+        "logprob": -1.5546875,
         "special": false,
-        "text": " the"
+        "text": " different"
       },
       {
         "id": 3295,
-        "logprob": -0.8203125,
+        "logprob": -0.97265625,
         "special": false,
         "text": " color"
       }
     ],
     "top_tokens": null
   },
-  "generated_text": "blue, red, yellow, \nand blue colors. A number of the color"
+  "generated_text": "blue, red, yellow, \nand blue colors. A number of different color"
 }
diff --git a/integration-tests/models/test_mamba.py b/integration-tests/models/test_mamba.py
index 8548970a..85ed8fd1 100644
--- a/integration-tests/models/test_mamba.py
+++ b/integration-tests/models/test_mamba.py
@@ -47,7 +47,7 @@ async def test_mamba_all_params(fused_kernel_mamba, response_snapshot):
     assert response.details.generated_tokens == 10
     assert (
         response.generated_text
-        == "blue, red, yellow, \nand blue colors. A number of the color"
+        == "blue, red, yellow, \nand blue colors. A number of different color"
     )
     assert response == response_snapshot
 
@@ -62,6 +62,7 @@ async def test_mamba_load(
     )
 
     assert len(responses) == 4
+    assert responses[0].generated_text == "\n\nDeep learning is a new type of machine"
     assert all([r.generated_text == responses[0].generated_text for r in responses])
     assert responses[0].generated_text == "\n\nDeep learning is a new type of machine"
 
diff --git a/server/custom_kernels/setup.py b/server/custom_kernels/setup.py
index 69f6b72a..e0b83987 100644
--- a/server/custom_kernels/setup.py
+++ b/server/custom_kernels/setup.py
@@ -1,10 +1,7 @@
 from setuptools import setup
 from torch.utils.cpp_extension import BuildExtension, CUDAExtension
-import torch
 
 extra_compile_args = ["-std=c++17"]
-if not torch.version.hip:
-    extra_compile_args.append("-arch=compute_80")
 
 setup(
     name="custom_kernels",

From 38773453ae0d29fba3dc79a38d589ebdc5451093 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Dani=C3=ABl=20de=20Kok?= <me@danieldk.eu>
Date: Mon, 19 Aug 2024 09:28:38 +0200
Subject: [PATCH 56/72] nix: update to CUDA 12.4 (#2429)

* Update to CUDA 12.4

* poetry2nix: follow tgi-nix nixpkgs
---
 flake.lock | 47 +++++++++++++++++------------------------------
 flake.nix  |  5 ++++-
 2 files changed, 21 insertions(+), 31 deletions(-)

diff --git a/flake.lock b/flake.lock
index 7c772377..d0c2adbc 100644
--- a/flake.lock
+++ b/flake.lock
@@ -739,32 +739,16 @@
     },
     "nixpkgs_6": {
       "locked": {
-        "lastModified": 1719763542,
-        "narHash": "sha256-mXkOj9sJ0f69Nkc2dGGOWtof9d1YNY8Le/Hia3RN+8Q=",
-        "owner": "NixOS",
+        "lastModified": 1723912943,
+        "narHash": "sha256-39F9GzyhxYcY3wTeKuEFWRJWcrGBosO4nf4xzMTWZX8=",
+        "owner": "danieldk",
         "repo": "nixpkgs",
-        "rev": "e6cdd8a11b26b4d60593733106042141756b54a3",
+        "rev": "b82cdca86dbb30013b76c4b55d48806476820a5c",
         "type": "github"
       },
       "original": {
-        "owner": "NixOS",
-        "ref": "nixos-unstable-small",
-        "repo": "nixpkgs",
-        "type": "github"
-      }
-    },
-    "nixpkgs_7": {
-      "locked": {
-        "lastModified": 1723418128,
-        "narHash": "sha256-k1pEqsnB6ikZyasXbtV6A9akPZMKlsyENPDUA6PXoJo=",
-        "owner": "nixos",
-        "repo": "nixpkgs",
-        "rev": "129f579cbb5b4c1ad258fd96bdfb78eb14802727",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nixos",
-        "ref": "nixos-unstable-small",
+        "owner": "danieldk",
+        "ref": "cuda-12.4",
         "repo": "nixpkgs",
         "type": "github"
       }
@@ -773,16 +757,19 @@
       "inputs": {
         "flake-utils": "flake-utils_7",
         "nix-github-actions": "nix-github-actions",
-        "nixpkgs": "nixpkgs_6",
+        "nixpkgs": [
+          "tgi-nix",
+          "nixpkgs"
+        ],
         "systems": "systems_8",
         "treefmt-nix": "treefmt-nix"
       },
       "locked": {
-        "lastModified": 1723512448,
-        "narHash": "sha256-VSTtxGKre1p6zd6ACuBmfDcR+BT9+ml8Y3KrSbfGFYU=",
+        "lastModified": 1723854676,
+        "narHash": "sha256-+BrHfNuXrqeE7PoV6xDaoh0joYiJkvTTCIV0fFR3THw=",
         "owner": "nix-community",
         "repo": "poetry2nix",
-        "rev": "ed52f844c4dd04dde45550c3189529854384124e",
+        "rev": "d650118bce34c0238b9b54f23f7f173f9e4db867",
         "type": "github"
       },
       "original": {
@@ -1048,14 +1035,14 @@
     "tgi-nix": {
       "inputs": {
         "flake-compat": "flake-compat_4",
-        "nixpkgs": "nixpkgs_7"
+        "nixpkgs": "nixpkgs_6"
       },
       "locked": {
-        "lastModified": 1723532088,
-        "narHash": "sha256-6h/P/BkFDw8txlikonKXp5IbluHSPhHJTQRftJLkbLQ=",
+        "lastModified": 1723973328,
+        "narHash": "sha256-q5FmW4YFQcRb6fXHnrxL0uno6xcw9dcg+pFBbVM1xeQ=",
         "owner": "danieldk",
         "repo": "tgi-nix",
-        "rev": "32335a37ce0f703bab901baf7b74eb11e9972d5f",
+        "rev": "d2038f36589a8a179834e5771ffd081620ba94c3",
         "type": "github"
       },
       "original": {
diff --git a/flake.nix b/flake.nix
index cf05746a..6bfe74e8 100644
--- a/flake.nix
+++ b/flake.nix
@@ -8,7 +8,10 @@
     tgi-nix.url = "github:danieldk/tgi-nix";
     nixpkgs.follows = "tgi-nix/nixpkgs";
     flake-utils.url = "github:numtide/flake-utils";
-    poetry2nix.url = "github:nix-community/poetry2nix";
+    poetry2nix = {
+      url = "github:nix-community/poetry2nix";
+      inputs.nixpkgs.follows = "tgi-nix/nixpkgs";
+    };
     rust-overlay = {
       url = "github:oxalica/rust-overlay";
       inputs.nixpkgs.follows = "tgi-nix/nixpkgs";

From b70ae0969f11bae03a3c6194fc8c592a1d8a65b3 Mon Sep 17 00:00:00 2001
From: Nicolas Patry <patry.nicolas@protonmail.com>
Date: Tue, 20 Aug 2024 11:15:30 +0200
Subject: [PATCH 57/72] Prefix caching  (#2402)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Prefix caching WIP

* Fixing prefix attention.

* Fixing flashinfer import.

* Fixing black.

* Fixing medusa (still wrong outputs, but functional).

* Just medusa values now.

* Fixing medusa without prefix caching.

* Fixing prefix caching.

* Medusa requires reshaping.

* Removing the logs.

* Remove router.nix

* Fixup:

- Remove logs
- Disable VLMs (they do not work)
- Disable prefix caching when user wants prefill logprobs.

* Update flake.lock

---------

Co-authored-by: Daniël de Kok <me@danieldk.eu>
---
 backends/v3/src/queue.rs                      |  13 +-
 backends/v3/src/radix.rs                      |   1 +
 flake.lock                                    |   6 +-
 flake.nix                                     |   1 +
 .../layers/attention/__init__.py              |   7 +-
 .../layers/attention/cuda.py                  |  27 +-
 .../{flash_infer.py => flashinfer.py}         |  76 ++++++
 .../text_generation_server/layers/medusa.py   |   2 +
 .../custom_modeling/flash_cohere_modeling.py  |   2 +
 .../custom_modeling/flash_dbrx_modeling.py    |   2 +
 .../flash_deepseek_v2_modeling.py             |   2 +
 .../custom_modeling/flash_gemma2_modeling.py  |   2 +
 .../custom_modeling/flash_gemma_modeling.py   |   2 +
 .../custom_modeling/flash_gpt2_modeling.py    |   2 +
 .../custom_modeling/flash_llama_modeling.py   |   2 +
 .../custom_modeling/flash_mistral_modeling.py |   2 +
 .../custom_modeling/flash_mixtral_modeling.py |   2 +
 .../custom_modeling/flash_neox_modeling.py    |   2 +
 .../custom_modeling/flash_phi_modeling.py     |   2 +
 .../custom_modeling/flash_qwen2_modeling.py   |   2 +
 .../custom_modeling/flash_rw_modeling.py      |   4 +
 .../flash_santacoder_modeling.py              |   2 +
 .../flash_starcoder2_modeling.py              |   2 +
 .../models/flash_causal_lm.py                 | 252 ++++++++++++++----
 .../text_generation_server/models/globals.py  |   6 +-
 .../models/vlm_causal_lm.py                   |  89 +++++--
 26 files changed, 405 insertions(+), 107 deletions(-)
 rename server/text_generation_server/layers/attention/{flash_infer.py => flashinfer.py} (65%)

diff --git a/backends/v3/src/queue.rs b/backends/v3/src/queue.rs
index 0fb05a98..faa57c11 100644
--- a/backends/v3/src/queue.rs
+++ b/backends/v3/src/queue.rs
@@ -316,10 +316,15 @@ impl State {
                         + self.speculate
                         - 1;
 
-                    match block_allocator
-                        .allocate(tokens, entry.request.input_ids.clone())
-                        .await
-                    {
+                    // If users wants the prefill logprobs, we cannot reuse the cache.
+                    // So no input_ids for the radix tree.
+                    let input_ids = if entry.request.decoder_input_details {
+                        None
+                    } else {
+                        entry.request.input_ids.clone()
+                    };
+
+                    match block_allocator.allocate(tokens, input_ids).await {
                         None => {
                             // Entry is over budget
                             // Add it back to the front
diff --git a/backends/v3/src/radix.rs b/backends/v3/src/radix.rs
index ef963532..5bac1a31 100644
--- a/backends/v3/src/radix.rs
+++ b/backends/v3/src/radix.rs
@@ -205,6 +205,7 @@ pub struct RadixTrie {
     /// call that a real time lookup would require.
     time: u64,
 }
+
 impl Default for RadixTrie {
     fn default() -> Self {
         Self::new()
diff --git a/flake.lock b/flake.lock
index d0c2adbc..cd5d6d2a 100644
--- a/flake.lock
+++ b/flake.lock
@@ -900,11 +900,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1723515680,
-        "narHash": "sha256-nHdKymsHCVIh0Wdm4MvSgxcTTg34FJIYHRQkQYaSuvk=",
+        "lastModified": 1723602049,
+        "narHash": "sha256-Z/noCSn9WPkv7O77dWKLcBxe4Ub4bWyNzsL5JhjaQfw=",
         "owner": "oxalica",
         "repo": "rust-overlay",
-        "rev": "4ee3d9e9569f70d7bb40f28804d6fe950c81eab3",
+        "rev": "ea0bf33a11a26a62c60123c49d96011da396602c",
         "type": "github"
       },
       "original": {
diff --git a/flake.nix b/flake.nix
index 6bfe74e8..299e6b3d 100644
--- a/flake.nix
+++ b/flake.nix
@@ -84,6 +84,7 @@
                 grpcio-status
                 grpcio-tools
                 hf-transfer
+                ipdb
                 loguru
                 mamba-ssm
                 marlin-kernels
diff --git a/server/text_generation_server/layers/attention/__init__.py b/server/text_generation_server/layers/attention/__init__.py
index f9b1715e..56fc5319 100644
--- a/server/text_generation_server/layers/attention/__init__.py
+++ b/server/text_generation_server/layers/attention/__init__.py
@@ -6,7 +6,12 @@ from .common import Seqlen
 if os.getenv("USE_FLASH_ATTENTION", "").lower() == "false":
     raise ImportError("`USE_FLASH_ATTENTION` is false.")
 if SYSTEM == "cuda":
-    from .cuda import attention, paged_attention, reshape_and_cache, SUPPORTS_WINDOWING
+    from .cuda import (
+        attention,
+        paged_attention,
+        reshape_and_cache,
+        SUPPORTS_WINDOWING,
+    )
 elif SYSTEM == "rocm":
     from .rocm import attention, paged_attention, reshape_and_cache, SUPPORTS_WINDOWING
 elif SYSTEM == "ipex":
diff --git a/server/text_generation_server/layers/attention/cuda.py b/server/text_generation_server/layers/attention/cuda.py
index 8703eb94..b3b7ea4f 100644
--- a/server/text_generation_server/layers/attention/cuda.py
+++ b/server/text_generation_server/layers/attention/cuda.py
@@ -76,7 +76,7 @@ def paged_attention(
     # sequences or heads is large, we use V1 since there is enough work
     # to parallelize.
     if ATTENTION == "flashinfer":
-        from text_generation_server.layers.attention.flash_infer import decode_state
+        from text_generation_server.layers.attention.flashinfer import decode_state
 
         return decode_state.get().forward(
             query.contiguous(),
@@ -221,9 +221,11 @@ SUPPORTS_WINDOWING = V2
 if ATTENTION == "flashinfer":
 
     def attention(
-        q,
-        k,
-        v,
+        q: torch.Tensor,
+        k: torch.Tensor,
+        v: torch.Tensor,
+        key_cache: torch.Tensor,
+        value_cache: torch.Tensor,
         cu_seqlens,
         max_s,
         softmax_scale,
@@ -231,14 +233,15 @@ if ATTENTION == "flashinfer":
         causal=True,
         softcap=0.0,
     ):
-        from text_generation_server.layers.attention.flash_infer import prefill_state
+        assert window_size_left == -1, "Windowing is not supported with flash infer"
+        from text_generation_server.layers.attention.flashinfer import (
+            prefill_with_paged_kv_state,
+        )
 
-        return prefill_state.get().forward(
-            q,
-            k,
-            v,
+        return prefill_with_paged_kv_state.get().forward(
+            q.contiguous(),
             causal=causal,
-            window_left=window_size_left,
+            paged_kv_cache=(key_cache, value_cache),
             logits_soft_cap=softcap,
             sm_scale=softmax_scale,
         )
@@ -249,6 +252,8 @@ elif V2:
         q,
         k,
         v,
+        key_cache: torch.Tensor,
+        value_cache: torch.Tensor,
         cu_seqlens,
         max_s,
         softmax_scale,
@@ -289,6 +294,8 @@ else:
         q,
         k,
         v,
+        key_cache: torch.Tensor,
+        value_cache: torch.Tensor,
         cu_seqlens,
         max_s,
         softmax_scale,
diff --git a/server/text_generation_server/layers/attention/flash_infer.py b/server/text_generation_server/layers/attention/flashinfer.py
similarity index 65%
rename from server/text_generation_server/layers/attention/flash_infer.py
rename to server/text_generation_server/layers/attention/flashinfer.py
index 56b53b2c..e1ef62c5 100644
--- a/server/text_generation_server/layers/attention/flash_infer.py
+++ b/server/text_generation_server/layers/attention/flashinfer.py
@@ -9,6 +9,10 @@ prefill_state: ContextVar[flashinfer.BatchPrefillWithRaggedKVCacheWrapper] = Con
     "prefill_state"
 )
 
+prefill_with_paged_kv_state: ContextVar[
+    flashinfer.BatchPrefillWithPagedKVCacheWrapper
+] = ContextVar("prefill_with_paged_kv_state")
+
 decode_state: ContextVar[flashinfer.BatchDecodeWithPagedKVCacheWrapper] = ContextVar(
     "decode_state"
 )
@@ -24,6 +28,78 @@ def get_workspace(device):
     return workspace
 
 
+def create_prefill_with_paged_kv_state(
+    *,
+    device: torch.device,
+):
+    """Create a prefill state that uses the KV cache."""
+    workspace_buffer = get_workspace(device)
+    return flashinfer.BatchPrefillWithPagedKVCacheWrapper(
+        workspace_buffer, kv_layout="NHD", use_cuda_graph=False
+    )
+
+
+@contextmanager
+def use_prefill_with_paged_kv_state(
+    *,
+    state: flashinfer.BatchPrefillWithPagedKVCacheWrapper,
+    block_tables: torch.Tensor,
+    cu_seqlens: torch.Tensor,
+    input_lengths: torch.Tensor,
+    num_heads: int,
+    num_kv_heads: int,
+    head_size: int,
+    page_size: int,
+    query_dtype: str = "float16",
+):
+    """
+    Context manager to set the active flashinfer prefill state to the given
+    `state` and parameters. This state will be used by all calls to the
+    `attention` function while the context manager is active.
+    """
+
+    indptr = torch.zeros(
+        input_lengths.shape[0] + 1, device=input_lengths.device, dtype=torch.int32
+    )
+    # Round up to page size and then calculate the cumulative sum to get
+    # the indices into the block table.
+    torch.add(input_lengths, page_size - 1, out=indptr[1:])
+    indptr[1:].div_(page_size, rounding_mode="floor")
+    indptr[1:].cumsum_(-1)
+
+    # Get the lengths of the last page in a block.
+    if page_size == 1:
+        last_page_len = torch.ones(
+            input_lengths.shape[0], dtype=torch.int32, device=input_lengths.device
+        )
+    else:
+        last_page_len = torch.empty(
+            input_lengths.shape[0], dtype=torch.int32, device=input_lengths.device
+        )
+        torch.sub(input_lengths, 1, out=last_page_len)
+        last_page_len.remainder_(page_size)
+        last_page_len += 1
+
+    token = prefill_with_paged_kv_state.set(state)
+    try:
+        state.begin_forward(
+            qo_indptr=cu_seqlens,
+            paged_kv_indptr=indptr,
+            paged_kv_indices=block_tables,
+            paged_kv_last_page_len=last_page_len,
+            num_qo_heads=num_heads,
+            num_kv_heads=num_kv_heads,
+            head_dim=head_size,
+            q_data_type=query_dtype,
+            page_size=page_size,
+        )
+        yield
+    finally:
+        state.end_forward()
+        if token is not None:
+            prefill_with_paged_kv_state.reset(token)
+
+
 def create_prefill_state(
     *,
     device: torch.device,
diff --git a/server/text_generation_server/layers/medusa.py b/server/text_generation_server/layers/medusa.py
index 7579ccdb..139c4dc2 100644
--- a/server/text_generation_server/layers/medusa.py
+++ b/server/text_generation_server/layers/medusa.py
@@ -32,6 +32,8 @@ class MedusaModel(torch.nn.Module):
         )
 
     def forward(self, x):
+        if not self.heads:
+            return None
         speculative_logits = torch.stack([head(x) for head in self.heads], dim=1)
         return speculative_logits
 
diff --git a/server/text_generation_server/models/custom_modeling/flash_cohere_modeling.py b/server/text_generation_server/models/custom_modeling/flash_cohere_modeling.py
index e02a31d9..1eb8c6c3 100644
--- a/server/text_generation_server/models/custom_modeling/flash_cohere_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/flash_cohere_modeling.py
@@ -298,6 +298,8 @@ class FlashCohereAttention(torch.nn.Module):
                 query,
                 key,
                 value,
+                kv_cache[0],
+                kv_cache[1],
                 cu_seqlen_prefill,
                 max_s,
                 self.softmax_scale,
diff --git a/server/text_generation_server/models/custom_modeling/flash_dbrx_modeling.py b/server/text_generation_server/models/custom_modeling/flash_dbrx_modeling.py
index d3d1d1ef..fc0dca5b 100644
--- a/server/text_generation_server/models/custom_modeling/flash_dbrx_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/flash_dbrx_modeling.py
@@ -337,6 +337,8 @@ class DbrxAttention(torch.nn.Module):
                 query,
                 torch.select(kv, dim=1, index=0),
                 torch.select(kv, dim=1, index=1),
+                kv_cache[0],
+                kv_cache[1],
                 cu_seqlen_prefill,
                 max_s,
                 self.softmax_scale,
diff --git a/server/text_generation_server/models/custom_modeling/flash_deepseek_v2_modeling.py b/server/text_generation_server/models/custom_modeling/flash_deepseek_v2_modeling.py
index 0905d3c2..b25becd5 100644
--- a/server/text_generation_server/models/custom_modeling/flash_deepseek_v2_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/flash_deepseek_v2_modeling.py
@@ -365,6 +365,8 @@ class DeepseekV2Attention(torch.nn.Module):
                 query,
                 key,
                 value,
+                kv_cache[0],
+                kv_cache[1],
                 cu_seqlen_prefill,
                 max_s,
                 self.softmax_scale,
diff --git a/server/text_generation_server/models/custom_modeling/flash_gemma2_modeling.py b/server/text_generation_server/models/custom_modeling/flash_gemma2_modeling.py
index 54d212e6..faf0f325 100644
--- a/server/text_generation_server/models/custom_modeling/flash_gemma2_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/flash_gemma2_modeling.py
@@ -238,6 +238,8 @@ class FlashGemma2Attention(torch.nn.Module):
                 query,
                 torch.select(kv, dim=1, index=0),
                 torch.select(kv, dim=1, index=1),
+                kv_cache[0],
+                kv_cache[1],
                 cu_seqlen_prefill,
                 max_s,
                 self.softmax_scale,
diff --git a/server/text_generation_server/models/custom_modeling/flash_gemma_modeling.py b/server/text_generation_server/models/custom_modeling/flash_gemma_modeling.py
index 178efadb..33738a59 100644
--- a/server/text_generation_server/models/custom_modeling/flash_gemma_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/flash_gemma_modeling.py
@@ -232,6 +232,8 @@ class FlashGemmaAttention(torch.nn.Module):
                 query,
                 torch.select(kv, dim=1, index=0),
                 torch.select(kv, dim=1, index=1),
+                kv_cache[0],
+                kv_cache[1],
                 cu_seqlen_prefill,
                 max_s,
                 self.softmax_scale,
diff --git a/server/text_generation_server/models/custom_modeling/flash_gpt2_modeling.py b/server/text_generation_server/models/custom_modeling/flash_gpt2_modeling.py
index a19cff8c..d30b5a0a 100644
--- a/server/text_generation_server/models/custom_modeling/flash_gpt2_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/flash_gpt2_modeling.py
@@ -232,6 +232,8 @@ class FlashGPT2Attention(torch.nn.Module):
                 query,
                 key,
                 value,
+                kv_cache[0],
+                kv_cache[1],
                 cu_seqlen_prefill,
                 max_s,
                 self.softmax_scale,
diff --git a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py
index 9ea19a87..3253d2dc 100644
--- a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py
@@ -220,6 +220,8 @@ class FlashLlamaAttention(torch.nn.Module):
                 query,
                 torch.select(kv, dim=1, index=0),
                 torch.select(kv, dim=1, index=1),
+                kv_cache[0],
+                kv_cache[1],
                 cu_seqlen_prefill,
                 max_s,
                 self.softmax_scale,
diff --git a/server/text_generation_server/models/custom_modeling/flash_mistral_modeling.py b/server/text_generation_server/models/custom_modeling/flash_mistral_modeling.py
index dda53ff3..5a150267 100644
--- a/server/text_generation_server/models/custom_modeling/flash_mistral_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/flash_mistral_modeling.py
@@ -219,6 +219,8 @@ class MistralAttention(torch.nn.Module):
                 query,
                 torch.select(kv, dim=1, index=0),
                 torch.select(kv, dim=1, index=1),
+                kv_cache[0],
+                kv_cache[1],
                 cu_seqlen_prefill,
                 max_s,
                 self.softmax_scale,
diff --git a/server/text_generation_server/models/custom_modeling/flash_mixtral_modeling.py b/server/text_generation_server/models/custom_modeling/flash_mixtral_modeling.py
index 85431c6c..ad426ffe 100644
--- a/server/text_generation_server/models/custom_modeling/flash_mixtral_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/flash_mixtral_modeling.py
@@ -276,6 +276,8 @@ class MixtralAttention(torch.nn.Module):
                 query,
                 torch.select(kv, dim=1, index=0),
                 torch.select(kv, dim=1, index=1),
+                kv_cache[0],
+                kv_cache[1],
                 cu_seqlen_prefill,
                 max_s,
                 self.softmax_scale,
diff --git a/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py b/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py
index 67237d5c..b684e035 100644
--- a/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py
@@ -173,6 +173,8 @@ class FlashNeoxAttention(torch.nn.Module):
                 qkv[:, 0],
                 qkv[:, 1],
                 qkv[:, 2],
+                kv_cache[0],
+                kv_cache[1],
                 cu_seqlen_prefill,
                 max_s,
                 self.softmax_scale,
diff --git a/server/text_generation_server/models/custom_modeling/flash_phi_modeling.py b/server/text_generation_server/models/custom_modeling/flash_phi_modeling.py
index a9e18348..efe27c13 100644
--- a/server/text_generation_server/models/custom_modeling/flash_phi_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/flash_phi_modeling.py
@@ -194,6 +194,8 @@ class FlashPhiAttention(torch.nn.Module):
                 query,
                 torch.select(kv, dim=1, index=0),
                 torch.select(kv, dim=1, index=1),
+                kv_cache[0],
+                kv_cache[1],
                 cu_seqlen_prefill,
                 max_s,
                 self.softmax_scale,
diff --git a/server/text_generation_server/models/custom_modeling/flash_qwen2_modeling.py b/server/text_generation_server/models/custom_modeling/flash_qwen2_modeling.py
index 865cc85d..879b8abd 100644
--- a/server/text_generation_server/models/custom_modeling/flash_qwen2_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/flash_qwen2_modeling.py
@@ -137,6 +137,8 @@ class Qwen2Attention(torch.nn.Module):
                 query,
                 torch.select(kv, dim=1, index=0),
                 torch.select(kv, dim=1, index=1),
+                kv_cache[0],
+                kv_cache[1],
                 cu_seqlen_prefill,
                 max_s,
                 self.softmax_scale,
diff --git a/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py b/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py
index 10f995a3..c72a9b90 100644
--- a/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py
@@ -208,6 +208,8 @@ class FlashRWAttention(torch.nn.Module):
                 query,
                 torch.select(kv, dim=1, index=0),
                 torch.select(kv, dim=1, index=1),
+                kv_cache[0],
+                kv_cache[1],
                 cu_seqlen_prefill,
                 max_s,
                 self.softmax_scale,
@@ -326,6 +328,8 @@ class FlashRWLargeAttention(torch.nn.Module):
                 query,
                 torch.select(kv, dim=2, index=0),
                 torch.select(kv, dim=2, index=1),
+                kv_cache[0],
+                kv_cache[1],
                 cu_seqlen_prefill,
                 max_s,
                 self.softmax_scale,
diff --git a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py
index c2676782..109304be 100644
--- a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py
@@ -293,6 +293,8 @@ class FlashMQAttention(torch.nn.Module):
                 query,
                 torch.select(key_value, dim=1, index=0),
                 torch.select(key_value, dim=1, index=1),
+                kv_cache[0],
+                kv_cache[1],
                 cu_seqlen_prefill,
                 max_s,
                 self.softmax_scale,
diff --git a/server/text_generation_server/models/custom_modeling/flash_starcoder2_modeling.py b/server/text_generation_server/models/custom_modeling/flash_starcoder2_modeling.py
index e562eb89..200d4ef0 100644
--- a/server/text_generation_server/models/custom_modeling/flash_starcoder2_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/flash_starcoder2_modeling.py
@@ -242,6 +242,8 @@ class Starcoder2Attention(torch.nn.Module):
                 query,
                 torch.select(kv, dim=1, index=0),
                 torch.select(kv, dim=1, index=1),
+                kv_cache[0],
+                kv_cache[1],
                 cu_seqlen_prefill,
                 max_s,
                 self.softmax_scale,
diff --git a/server/text_generation_server/models/flash_causal_lm.py b/server/text_generation_server/models/flash_causal_lm.py
index 5e2fd20a..dd4203e0 100644
--- a/server/text_generation_server/models/flash_causal_lm.py
+++ b/server/text_generation_server/models/flash_causal_lm.py
@@ -43,6 +43,7 @@ from text_generation_server.models.globals import (
     ATTENTION,
     BLOCK_SIZE,
     CUDA_GRAPHS,
+    PREFIX_CACHING,
     get_adapter_to_index,
 )
 from text_generation_server.layers.attention import Seqlen
@@ -138,6 +139,9 @@ class FlashCausalLMBatch(Batch):
     block_tables_tensor: torch.Tensor
     # tensor of length \sum_{i=0}^{b} max_s_i  holding the paged attention slots for all sequences
     slots: torch.Tensor
+    # size [b], containing the number of blocks that can be retrieved from the cache
+    prefix_lens: List[int]
+    prefix_lens_tensor: torch.Tensor
 
     max_seqlen: int
 
@@ -146,6 +150,9 @@ class FlashCausalLMBatch(Batch):
     prefill_next_token_indices: Optional[torch.tensor]
     prefill_cu_outlens: Optional[List[int]]
 
+    # Prefixes
+    prefix_ids: List[List[int]]
+
     # All tokens
     all_input_ids: List[List[int]]
     all_input_ids_tensor: torch.Tensor
@@ -213,6 +220,7 @@ class FlashCausalLMBatch(Batch):
         prefix_offsets = []
         read_offsets = []
         all_input_ids = []
+        prefix_ids = []
         requests_idx_mapping = {}
 
         all_prefill_logprobs = True
@@ -230,7 +238,7 @@ class FlashCausalLMBatch(Batch):
 
         # Cumulative length
         cumulative_length = 0
-        cumulative_max_length = 0
+        cumulative_slot_tokens = 0
         prefill_out_cumulative_length = 0
 
         num_blocks = 0
@@ -240,6 +248,7 @@ class FlashCausalLMBatch(Batch):
 
         block_tables = []
         slots = []
+        prefix_lens = []
 
         # Parse batch
         for i, (r, tokenized_input) in enumerate(
@@ -255,6 +264,19 @@ class FlashCausalLMBatch(Batch):
             ):
                 tokenized_input = tokenized_input[1:]
 
+            orig_input_length = len(tokenized_input)
+
+            if PREFIX_CACHING:
+                prefix_len = r.prefix_len
+                if prefix_len == orig_input_length:
+                    assert prefix_len > 0
+                    prefix_len -= 1
+            else:
+                prefix_len = 0
+
+            prefix_ids.append(tokenized_input[:prefix_len])
+            tokenized_input = tokenized_input[prefix_len:]
+
             input_length = len(tokenized_input)
             input_lengths.append(input_length)
 
@@ -264,7 +286,9 @@ class FlashCausalLMBatch(Batch):
             all_input_ids.append(tokenized_input)
 
             # Position ids
-            request_position_ids = torch.arange(0, input_length, dtype=torch.int32)
+            request_position_ids = torch.arange(
+                prefix_len, orig_input_length, dtype=torch.int32
+            )
             position_ids.append(request_position_ids)
 
             # Add cumulative lengths of all previous inputs
@@ -288,11 +312,17 @@ class FlashCausalLMBatch(Batch):
             # Remove one as the first token des not have a past
             speculative_length = get_speculate()
             speculative_length = 0 if speculative_length is None else speculative_length
-            total_tokens = input_length + max_new_tokens - 1 + speculative_length
+
+            # Tokens that need to be mapped to blocks.
+            block_tokens = orig_input_length + max_new_tokens - 1 + speculative_length
+
+            # Tokens that need to be mapped to slots. We don't need slots for the
+            # cached prefix (if present).
+            slot_tokens = input_length + max_new_tokens - 1 + speculative_length
 
             # blocks and slots can be empty (for example in warmup)
             if not r.blocks:
-                needed_blocks = math.ceil(total_tokens / BLOCK_SIZE)
+                needed_blocks = math.ceil(block_tokens / BLOCK_SIZE)
                 request_blocks = [
                     b for b in range(num_blocks, num_blocks + needed_blocks)
                 ]
@@ -303,16 +333,20 @@ class FlashCausalLMBatch(Batch):
                 ]
             else:
                 request_blocks = r.blocks
-                request_slots = r.slots
+                request_slots = r.slots[
+                    prefix_len:  #: orig_input_length + max_new_tokens + speculative_length
+                ]
 
             block_tables.append(request_blocks)
-            slots.extend(request_slots[:total_tokens])
+
+            slots.extend(request_slots)
+            prefix_lens.append(prefix_len)
             num_blocks += len(request_blocks)
-            start_slots.append(cumulative_max_length)
+            start_slots.append(cumulative_slot_tokens)
 
             request_slot_indices = torch.arange(
-                cumulative_max_length,
-                cumulative_max_length + input_length,
+                cumulative_slot_tokens,
+                cumulative_slot_tokens + input_length,
                 dtype=torch.int64,
             )
             slot_indices.append(request_slot_indices)
@@ -348,7 +382,7 @@ class FlashCausalLMBatch(Batch):
 
             # Update
             cumulative_length += input_length
-            cumulative_max_length += total_tokens
+            cumulative_slot_tokens += slot_tokens
             max_seqlen = max(max_seqlen, input_length)
             max_blocks = max(max_blocks, len(request_blocks))
             max_length = max(
@@ -425,12 +459,14 @@ class FlashCausalLMBatch(Batch):
         )
 
         slots = torch.tensor(slots, dtype=torch.int64, device=device)
+
         block_tables_tensor = torch.zeros(
             (len(block_tables), max_blocks), dtype=torch.int32, device="cpu"
         )
         for i, request_blocks in enumerate(block_tables):
             block_tables_tensor[i, : len(request_blocks)] = torch.tensor(request_blocks)
         block_tables_tensor = block_tables_tensor.to(device)
+        prefix_lens_tensor = torch.tensor(prefix_lens, dtype=torch.int32, device=device)
 
         return cls(
             batch_id=pb.id,
@@ -445,6 +481,8 @@ class FlashCausalLMBatch(Batch):
             block_tables=block_tables,
             block_tables_tensor=block_tables_tensor,
             slots=slots,
+            prefix_lens=prefix_lens,
+            prefix_lens_tensor=prefix_lens_tensor,
             max_seqlen=max_seqlen,
             prefill_head_indices=prefill_head_indices,
             prefill_next_token_indices=prefill_next_token_indices,
@@ -455,6 +493,7 @@ class FlashCausalLMBatch(Batch):
             read_offsets=read_offsets,
             all_input_ids=all_input_ids,
             all_input_ids_tensor=all_input_ids_tensor,
+            prefix_ids=prefix_ids,
             next_token_chooser=next_token_chooser,
             stopping_criterias=stopping_criterias,
             top_n_tokens=top_n_tokens,
@@ -510,8 +549,10 @@ class FlashCausalLMBatch(Batch):
         start_slots = []
         block_tables = []
         all_input_ids = []
+        prefix_ids = []
 
         input_lengths = []
+        prefix_lens = []
         prefix_offsets = []
         read_offsets = []
 
@@ -533,11 +574,14 @@ class FlashCausalLMBatch(Batch):
 
             # Get length
             request_input_length = self.input_lengths[idx]
+            prefix_len = self.prefix_lens[idx]
             max_seqlen = max(max_seqlen, request_input_length)
 
             all_input_ids.append(self.all_input_ids[idx])
+            prefix_ids.append(self.prefix_ids[idx])
 
             input_lengths.append(request_input_length)
+            prefix_lens.append(prefix_len)
             prefix_offsets.append(self.prefix_offsets[idx])
             read_offsets.append(self.read_offsets[idx])
 
@@ -582,6 +626,7 @@ class FlashCausalLMBatch(Batch):
         block_tables_tensor = self.block_tables_tensor[indices]
         input_lengths_tensor = self.input_lengths_tensor[indices]
         slots = self.slots[slot_filtering_indices]
+        prefix_lens_tensor = self.prefix_lens_tensor[indices]
         next_token_chooser = self.next_token_chooser.filter(indices)
         top_n_tokens_tensor = self.top_n_tokens_tensor[indices]
         speculative_ids = (
@@ -617,10 +662,13 @@ class FlashCausalLMBatch(Batch):
             prefill_cu_outlens=None,
             input_lengths=input_lengths,
             input_lengths_tensor=input_lengths_tensor,
+            prefix_lens=prefix_lens,
+            prefix_lens_tensor=prefix_lens_tensor,
             prefix_offsets=prefix_offsets,
             read_offsets=read_offsets,
             all_input_ids=all_input_ids,
             all_input_ids_tensor=all_input_ids_tensor,
+            prefix_ids=prefix_ids,
             next_token_chooser=next_token_chooser,
             stopping_criterias=stopping_criterias,
             top_n_tokens=top_n_tokens,
@@ -681,6 +729,7 @@ class FlashCausalLMBatch(Batch):
         block_tables_tensor = batches[0].block_tables_tensor.new_zeros(
             (total_batch_size, max_blocks)
         )
+        prefix_lens_tensor = batches[0].prefix_lens_tensor.new_empty(total_batch_size)
         all_input_ids_tensor = batches[0].all_input_ids_tensor.new_zeros(
             (total_batch_size, max_length)
         )
@@ -698,7 +747,9 @@ class FlashCausalLMBatch(Batch):
 
         start_slots = []
         block_tables = []
+        prefix_lens = []
         all_input_ids = []
+        prefix_ids = []
 
         input_lengths = []
         prefix_offsets = []
@@ -760,10 +811,14 @@ class FlashCausalLMBatch(Batch):
                 start_index:end_index, : batch.block_tables_tensor.shape[1]
             ] = batch.block_tables_tensor[:, :max_blocks]
 
+            prefix_lens_tensor[start_index:end_index] = batch.prefix_lens_tensor
+
             start_slots.append(batch.start_slots + cumulative_slots)
 
             block_tables.extend(batch.block_tables)
+            prefix_lens.extend(batch.prefix_lens)
             all_input_ids.extend(batch.all_input_ids)
+            prefix_ids.extend(batch.prefix_ids)
 
             input_lengths.extend(batch.input_lengths)
             prefix_offsets.extend(batch.prefix_offsets)
@@ -809,6 +864,8 @@ class FlashCausalLMBatch(Batch):
             slot_indices=slot_indices,
             block_tables=block_tables,
             block_tables_tensor=block_tables_tensor,
+            prefix_lens=prefix_lens,
+            prefix_lens_tensor=prefix_lens_tensor,
             slots=slots,
             max_seqlen=max_seqlen,
             prefill_head_indices=None,
@@ -820,6 +877,7 @@ class FlashCausalLMBatch(Batch):
             read_offsets=read_offsets,
             all_input_ids=all_input_ids,
             all_input_ids_tensor=all_input_ids_tensor,
+            prefix_ids=prefix_ids,
             next_token_chooser=next_token_chooser,
             stopping_criterias=stopping_criterias,
             top_n_tokens=top_n_tokens,
@@ -970,19 +1028,22 @@ class FlashCausalLM(Model):
         self.kv_cache = []
 
         if ATTENTION == "flashinfer":
-            from text_generation_server.layers.attention.flash_infer import (
+            from text_generation_server.layers.attention.flashinfer import (
                 create_prefill_state,
                 create_decode_state,
+                create_prefill_with_paged_kv_state,
             )
 
             self.prefill_state = create_prefill_state(device=device)
+            self.prefill_with_paged_kv_state = create_prefill_with_paged_kv_state(
+                device=device
+            )
 
-            if not CUDA_GRAPHS:
-                self.decode_state = create_decode_state(
-                    device=device,
-                    num_heads=self.num_heads,
-                    num_kv_heads=self.num_kv_heads,
-                )
+            self.decode_state = create_decode_state(
+                device=device,
+                num_heads=self.num_heads,
+                num_kv_heads=self.num_kv_heads,
+            )
 
         super().__init__(
             model_id=model_id,
@@ -1074,12 +1135,23 @@ class FlashCausalLM(Model):
         input_ids = torch.zeros(bs, dtype=torch.int64, device=self.device)
         position_ids = torch.zeros(bs, dtype=torch.int32, device=self.device)
         slots = torch.arange(bs, dtype=torch.int64, device=self.device)
-        input_lengths = torch.ones(bs, dtype=torch.int32, device=self.device) * max_s
-        block_tables = (
-            torch.arange(max_bt, dtype=torch.int32, device=self.device)
-            .repeat(bs)
-            .reshape((bs, max_bt))
+        input_lengths = [max_s] * bs
+        prefix_lengths = [0] * bs
+        input_lengths_tensor = (
+            torch.ones(bs, dtype=torch.int32, device=self.device) * max_s
         )
+        prefix_lengths_tensor = torch.zeros(bs, dtype=torch.int32, device=self.device)
+        block_tables = torch.arange(
+            max_bt, dtype=torch.int32, device=self.device
+        ).repeat(bs)
+        block_tables = block_tables.reshape((bs, max_bt))
+
+        if ATTENTION == "flashinfer":
+            block_tables = block_tables_to_ragged(
+                block_tables=block_tables,
+                input_lengths=input_lengths,
+                prefix_lens=prefix_lengths,
+            )
 
         self.cuda_graphs[bs] = {
             "input_ids": input_ids,
@@ -1087,14 +1159,14 @@ class FlashCausalLM(Model):
             "kv_cache": self.kv_cache,
             "block_tables": block_tables,
             "slots": slots,
-            "input_lengths": input_lengths,
+            "input_lengths": input_lengths_tensor,
         }
-        input_lengths_ = Seqlen(input_lengths=input_lengths)
+        input_lengths_ = Seqlen(input_lengths=input_lengths_tensor)
         graph = torch.cuda.CUDAGraph()
         self.cuda_graphs[bs]["graph"] = graph
 
         if ATTENTION == "flashinfer":
-            from text_generation_server.layers.attention.flash_infer import (
+            from text_generation_server.layers.attention.flashinfer import (
                 create_decode_state_cuda_graphs,
             )
 
@@ -1104,7 +1176,7 @@ class FlashCausalLM(Model):
             last_page_len = torch.ones(bs, dtype=torch.int32, device=self.device)
             state = create_decode_state_cuda_graphs(
                 device=input_ids.device,
-                block_tables=block_tables.view(-1),
+                block_tables=block_tables,
                 block_tables_ptr=block_tables_ptr,
                 last_page_len=last_page_len,
                 num_heads=self.num_heads,
@@ -1120,7 +1192,10 @@ class FlashCausalLM(Model):
             block_tables=block_tables,
             cu_seqlen_prefill=None,
             input_lengths=input_lengths,
+            input_lengths_tensor=input_lengths_tensor,
             state=state,
+            prefix_lens=prefix_lengths,
+            prefix_lens_tensor=prefix_lengths_tensor,
         ):
             self.model.forward(
                 input_ids=input_ids,
@@ -1138,7 +1213,7 @@ class FlashCausalLM(Model):
             torch.cuda.synchronize()
 
             with torch.cuda.graph(graph, pool=MEM_POOL):
-                input_lengths = Seqlen(input_lengths=input_lengths)
+                input_lengths_tensor = Seqlen(input_lengths=input_lengths_tensor)
                 logits, speculative_logits = self.model.forward(
                     input_ids=input_ids,
                     position_ids=position_ids,
@@ -1146,7 +1221,7 @@ class FlashCausalLM(Model):
                     kv_cache=self.kv_cache,
                     block_tables=block_tables,
                     slots=slots,
-                    input_lengths=input_lengths,
+                    input_lengths=input_lengths_tensor,
                     max_s=max_s,
                     prefill_cache_indices=None,
                     lm_head_indices=None,
@@ -1334,6 +1409,9 @@ class FlashCausalLM(Model):
             input_lengths = (
                 input_lengths.unsqueeze(-1).expand(B, new_length) + arange_int
             ).view(-1)
+            prefix_lens_tensor = (
+                batch.prefix_lens_tensor.unsqueeze(-1).expand(B, new_length)
+            ).reshape(-1)
 
             # Add Copy the block tables for all members
             block_tables = (
@@ -1354,6 +1432,7 @@ class FlashCausalLM(Model):
             block_tables = batch.block_tables_tensor
             slots = batch.slots[batch.slot_indices]
             input_lengths = batch.input_lengths_tensor
+            prefix_lens_tensor = batch.prefix_lens_tensor
             max_s = batch.max_seqlen
             lm_head_indices = batch.prefill_head_indices
 
@@ -1372,10 +1451,20 @@ class FlashCausalLM(Model):
             cuda_graph = None
 
         if cu_seqlen_prefill is not None or cuda_graph is None:
+            input_lengths = input_lengths + prefix_lens_tensor
+            if PREFIX_CACHING:
+                block_tables = block_tables_to_ragged(
+                    block_tables=block_tables,
+                    input_lengths=batch.input_lengths,
+                    prefix_lens=batch.prefix_lens,
+                )
             with self._forward_context(
                 block_tables=block_tables,
                 cu_seqlen_prefill=cu_seqlen_prefill,
-                input_lengths=input_lengths,
+                input_lengths=batch.input_lengths,
+                input_lengths_tensor=input_lengths,
+                prefix_lens=batch.prefix_lens,
+                prefix_lens_tensor=prefix_lens_tensor,
             ):
                 input_lengths = Seqlen(input_lengths=input_lengths)
                 logits, speculative_logits = self.model.forward(
@@ -1399,20 +1488,32 @@ class FlashCausalLM(Model):
         # Static inputs are potentially padded
         cuda_graph["input_ids"][: input_ids.shape[0]] = input_ids
         cuda_graph["position_ids"][: position_ids.shape[0]] = position_ids
-        cuda_graph["block_tables"][
-            : block_tables.shape[0], : block_tables.shape[1]
-        ] = block_tables
+        if ATTENTION == "flashinfer":
+            block_tables = block_tables_to_ragged(
+                block_tables=block_tables,
+                input_lengths=batch.input_lengths,
+                prefix_lens=batch.prefix_lens,
+            )
+            cuda_graph["block_tables"][: block_tables.shape[0]] = block_tables
+        else:
+            cuda_graph["block_tables"][
+                : block_tables.shape[0], : block_tables.shape[1]
+            ] = block_tables
         cuda_graph["slots"].fill_(-1)
         cuda_graph["slots"][: slots.shape[0]] = slots
         cuda_graph["input_lengths"].zero_()
-        cuda_graph["input_lengths"][: input_lengths.shape[0]] = input_lengths
+        cuda_graph["input_lengths"][: input_lengths.shape[0]] = (
+            input_lengths + prefix_lens_tensor
+        )
 
-        state = cuda_graph.get("state")
         with self._forward_context(
-            block_tables=block_tables,
+            block_tables=cuda_graph["block_tables"],
             cu_seqlen_prefill=None,
-            input_lengths=input_lengths,
-            state=state,
+            input_lengths=batch.input_lengths,
+            input_lengths_tensor=cuda_graph["input_lengths"],
+            prefix_lens=batch.prefix_lens,
+            prefix_lens_tensor=prefix_lens_tensor,
+            state=cuda_graph.get("state"),
         ):
             # Replay the graph
             cuda_graph["graph"].replay()
@@ -1610,6 +1711,7 @@ class FlashCausalLM(Model):
             batch.read_offsets,
             batch.stopping_criterias,
             batch.all_input_ids,
+            batch.prefix_ids,
             batch.next_token_chooser.do_sample,
             batch.next_token_chooser.seeds,
             batch.top_n_tokens,
@@ -1627,6 +1729,7 @@ class FlashCausalLM(Model):
             read_offset,
             stopping_criteria,
             all_input_ids,
+            prefix_ids,
             do_sample,
             seed,
             top_n_tokens,
@@ -1701,18 +1804,18 @@ class FlashCausalLM(Model):
                     out_end_index = batch.prefill_cu_outlens[i + 1]
 
                     # Remove generated token to only have prefill and add nan for first prompt token
-                    request_prefill_logprobs = [float("nan")] + prefill_logprobs[
-                        out_start_index : out_end_index - 1
-                    ]
+                    request_prefill_logprobs = (
+                        [float("nan")] * (len(prefix_ids) + 1)
+                    ) + prefill_logprobs[out_start_index : out_end_index - 1]
                     prefill_token_ids = all_input_ids[:-1]
                     prefill_texts = self.tokenizer.batch_decode(
-                        prefill_token_ids,
+                        prefix_ids + prefill_token_ids,
                         clean_up_tokenization_spaces=False,
                         skip_special_tokens=False,
                     )
 
                     prefill_tokens = Tokens(
-                        prefill_token_ids,
+                        prefix_ids + prefill_token_ids,
                         request_prefill_logprobs,
                         prefill_texts,
                         is_special=[],
@@ -1794,33 +1897,68 @@ class FlashCausalLM(Model):
         *,
         block_tables: torch.Tensor,
         cu_seqlen_prefill: Optional[torch.Tensor],
-        input_lengths: torch.Tensor,
+        input_lengths: List[int],
+        input_lengths_tensor: torch.Tensor,
+        prefix_lens: List[int],
+        prefix_lens_tensor: torch.Tensor,
         state: Optional[Any] = None,
     ) -> ContextManager:
         if ATTENTION != "flashinfer":
             return nullcontext()
 
-        from text_generation_server.layers.attention.flash_infer import (
+        from text_generation_server.layers.attention.flashinfer import (
             use_decode_state,
-            use_prefill_state,
+            use_prefill_with_paged_kv_state,
         )
 
+        # has_prefix_lens = any(prefix_len > 0 for prefix_len in prefix_lens)
+
         if cu_seqlen_prefill is not None:
-            return use_prefill_state(
-                state=state if state is not None else self.prefill_state,
+            return use_prefill_with_paged_kv_state(
+                state=(
+                    state if state is not None else self.prefill_with_paged_kv_state
+                ),
+                # block_tables=block_tables_to_ragged(
+                #     block_tables=block_tables,
+                #     input_lengths=input_lengths,
+                #     prefix_lens=prefix_lens,
+                # ),
+                block_tables=block_tables,
                 cu_seqlens=cu_seqlen_prefill,
-                num_heads=self.num_heads,
-                num_kv_heads=self.num_kv_heads,
-                head_size=self.head_size,
-            )
-        else:
-            assert input_lengths is not None
-            return use_decode_state(
-                state=state if state is not None else self.decode_state,
-                input_lengths=input_lengths,
-                block_tables=block_tables.view(-1),
+                input_lengths=input_lengths_tensor,
                 num_heads=self.num_heads,
                 num_kv_heads=self.num_kv_heads,
                 head_size=self.head_size,
                 page_size=BLOCK_SIZE,
             )
+        else:
+            assert input_lengths_tensor is not None
+            return use_decode_state(
+                state=state if state is not None else self.decode_state,
+                input_lengths=input_lengths_tensor,
+                block_tables=block_tables,
+                num_heads=self.num_heads,
+                num_kv_heads=self.num_kv_heads,
+                head_size=self.head_size,
+                page_size=BLOCK_SIZE,
+            )
+
+
+def block_tables_to_ragged(
+    *, block_tables: torch.Tensor, input_lengths: List[int], prefix_lens: List[int]
+) -> torch.Tensor:
+    """Convert block table to ragged format compatible with FlashInfer."""
+    assert len(input_lengths) == len(prefix_lens)
+
+    total_len = sum(input_lengths) + sum(prefix_lens)
+    block_tables_ragged = torch.empty(
+        total_len, dtype=torch.int32, device=block_tables.device
+    )
+
+    offset = 0
+    for i, (input_length, prefix_len) in enumerate(zip(input_lengths, prefix_lens)):
+        seq_len = prefix_len + input_length
+        block_tables_ragged[offset : offset + seq_len] = block_tables[i][:seq_len]
+        offset += seq_len
+
+    return block_tables_ragged
diff --git a/server/text_generation_server/models/globals.py b/server/text_generation_server/models/globals.py
index abc35421..d5133f5e 100644
--- a/server/text_generation_server/models/globals.py
+++ b/server/text_generation_server/models/globals.py
@@ -5,9 +5,8 @@ from typing import Dict, Optional
 
 from text_generation_server.utils.log import log_master
 
-PREFIX_CACHING = os.getenv("USE_PREFIX_CACHING", False)
-log_master(logger.info, f"Using Attention = {PREFIX_CACHING}")
-
+PREFIX_CACHING = os.getenv("USE_PREFIX_CACHING", "0").lower() in {"1", "true"}
+log_master(logger.info, f"Using prefix caching = {PREFIX_CACHING}")
 ATTENTION = os.getenv("ATTENTION", "flashinfer" if PREFIX_CACHING else "paged")
 _expected = {"paged", "flashdecoding", "flashinfer"}
 assert (
@@ -29,7 +28,6 @@ elif ATTENTION == "flashinfer":
 else:
     BLOCK_SIZE = 16
 
-
 cuda_graphs = os.getenv("CUDA_GRAPHS")
 if cuda_graphs is not None:
     try:
diff --git a/server/text_generation_server/models/vlm_causal_lm.py b/server/text_generation_server/models/vlm_causal_lm.py
index 7de54aa4..2ed1a119 100644
--- a/server/text_generation_server/models/vlm_causal_lm.py
+++ b/server/text_generation_server/models/vlm_causal_lm.py
@@ -11,7 +11,9 @@ from text_generation_server.pb import generate_pb2
 from text_generation_server.models.flash_causal_lm import (
     FlashCausalLMBatch,
     FlashCausalLM,
+    block_tables_to_ragged,
 )
+from text_generation_server.models.globals import PREFIX_CACHING, ATTENTION
 from text_generation_server.utils.log import log_master
 from transformers import AutoProcessor
 from text_generation_server.layers.attention import Seqlen
@@ -254,6 +256,8 @@ class VlmCausalLM(FlashCausalLM):
         trust_remote_code: bool,
         **kwargs,
     ):
+        if PREFIX_CACHING:
+            raise NotImplementedError("Vlm do not work with prefix caching yet")
         if processor_kwargs is None:
             processor_kwargs = {}
         self.processor = processor_class.from_pretrained(
@@ -310,6 +314,9 @@ class VlmCausalLM(FlashCausalLM):
             input_lengths = (
                 input_lengths.unsqueeze(-1).expand(B, new_length) + arange_int
             ).view(-1)
+            prefix_lens_tensor = (
+                batch.prefix_lens_tensor.unsqueeze(-1).expand(B, new_length)
+            ).reshape(-1)
 
             # Add Copy the block tables for all members
             block_tables = (
@@ -330,6 +337,7 @@ class VlmCausalLM(FlashCausalLM):
             block_tables = batch.block_tables_tensor
             slots = batch.slots[batch.slot_indices]
             input_lengths = batch.input_lengths_tensor
+            prefix_lens_tensor = batch.prefix_lens_tensor
             max_s = batch.max_seqlen
             lm_head_indices = batch.prefill_head_indices
 
@@ -349,43 +357,68 @@ class VlmCausalLM(FlashCausalLM):
         else:
             cuda_graph = None
         if cu_seqlen_prefill is not None or cuda_graph is None:
-            input_lengths = Seqlen(input_lengths=input_lengths)
-            logits, speculative_logits = self.model.forward(
-                input_ids=input_ids,
-                position_ids=position_ids,
-                cu_seqlen_prefill=cu_seqlen_prefill,
-                kv_cache=kv_cache,
+            input_lengths = input_lengths + prefix_lens_tensor
+            if PREFIX_CACHING:
+                block_tables = block_tables_to_ragged(
+                    block_tables=block_tables,
+                    input_lengths=batch.input_lengths,
+                    prefix_lens=batch.prefix_lens,
+                )
+            with self._forward_context(
                 block_tables=block_tables,
-                slots=slots,
-                input_lengths=input_lengths,
-                max_s=max_s,
-                prefill_cache_indices=batch.prefill_cache_indices,
-                lm_head_indices=lm_head_indices,
-                pixel_values=batch.pixel_values,
-                pixel_attention_mask=batch.pixel_attention_mask,
-                image_sizes=batch.image_sizes,
-            )
-            if batch.prefill_cache_indices is not None:
-                batch.prefill_cache_indices = None
-            if batch.pixel_values is not None:
-                batch.pixel_values = None
-            if batch.pixel_attention_mask is not None:
-                batch.pixel_attention_mask = None
-            if batch.image_sizes is not None:
-                batch.image_sizes = None
-            return logits, speculative_logits
+                cu_seqlen_prefill=cu_seqlen_prefill,
+                input_lengths=batch.input_lengths,
+                input_lengths_tensor=input_lengths,
+                prefix_lens=batch.prefix_lens,
+                prefix_lens_tensor=prefix_lens_tensor,
+            ):
+                input_lengths = Seqlen(input_lengths=input_lengths)
+                logits, speculative_logits = self.model.forward(
+                    input_ids=input_ids,
+                    position_ids=position_ids,
+                    cu_seqlen_prefill=cu_seqlen_prefill,
+                    kv_cache=kv_cache,
+                    block_tables=block_tables,
+                    slots=slots,
+                    input_lengths=input_lengths,
+                    max_s=max_s,
+                    prefill_cache_indices=batch.prefill_cache_indices,
+                    lm_head_indices=lm_head_indices,
+                    pixel_values=batch.pixel_values,
+                    pixel_attention_mask=batch.pixel_attention_mask,
+                    image_sizes=batch.image_sizes,
+                )
+                if batch.prefill_cache_indices is not None:
+                    batch.prefill_cache_indices = None
+                if batch.pixel_values is not None:
+                    batch.pixel_values = None
+                if batch.pixel_attention_mask is not None:
+                    batch.pixel_attention_mask = None
+                if batch.image_sizes is not None:
+                    batch.image_sizes = None
+                return logits, speculative_logits
 
         # Copy inputs to the static inputs of the cuda graph
         # Static inputs are potentially padded
         cuda_graph["input_ids"][: input_ids.shape[0]] = input_ids
         cuda_graph["position_ids"][: position_ids.shape[0]] = position_ids
-        cuda_graph["block_tables"][
-            : block_tables.shape[0], : block_tables.shape[1]
-        ] = block_tables
+        if ATTENTION == "flashinfer":
+            block_tables = block_tables_to_ragged(
+                block_tables=block_tables,
+                input_lengths=batch.input_lengths,
+                prefix_lens=batch.prefix_lens,
+            )
+            cuda_graph["block_tables"][: block_tables.shape[0]] = block_tables
+        else:
+            cuda_graph["block_tables"][
+                : block_tables.shape[0], : block_tables.shape[1]
+            ] = block_tables
         cuda_graph["slots"].fill_(-1)
         cuda_graph["slots"][: slots.shape[0]] = slots
         cuda_graph["input_lengths"].zero_()
-        cuda_graph["input_lengths"][: input_lengths.shape[0]] = input_lengths
+        cuda_graph["input_lengths"][: input_lengths.shape[0]] = (
+            input_lengths + prefix_lens_tensor
+        )
 
         # Replay the graph
         cuda_graph["graph"].replay()

From f5f11b797e70b2232632d410273c5c4418475dd1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Dani=C3=ABl=20de=20Kok?= <me@danieldk.eu>
Date: Tue, 20 Aug 2024 22:07:33 +0200
Subject: [PATCH 58/72] nix: add pure server to flake, add both pure and impure
 devshells (#2430)

* nix: pure server and support both pure and impure devShells

* nix: remove unused poetry2nix input

It is not wired up and we now have a pure server.

* nix: add ipdb to impure devshell
---
 flake.lock     | 115 -------------------------------------------------
 flake.nix      |  64 +++++++++++----------------
 nix/server.nix | 105 ++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 129 insertions(+), 155 deletions(-)
 create mode 100644 nix/server.nix

diff --git a/flake.lock b/flake.lock
index cd5d6d2a..c20dd98a 100644
--- a/flake.lock
+++ b/flake.lock
@@ -492,24 +492,6 @@
         "type": "github"
       }
     },
-    "flake-utils_7": {
-      "inputs": {
-        "systems": "systems_7"
-      },
-      "locked": {
-        "lastModified": 1710146030,
-        "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=",
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a",
-        "type": "github"
-      },
-      "original": {
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "type": "github"
-      }
-    },
     "gitignore": {
       "inputs": {
         "nixpkgs": [
@@ -594,27 +576,6 @@
         "type": "github"
       }
     },
-    "nix-github-actions": {
-      "inputs": {
-        "nixpkgs": [
-          "poetry2nix",
-          "nixpkgs"
-        ]
-      },
-      "locked": {
-        "lastModified": 1703863825,
-        "narHash": "sha256-rXwqjtwiGKJheXB43ybM8NwWB8rO2dSRrEqes0S7F5Y=",
-        "owner": "nix-community",
-        "repo": "nix-github-actions",
-        "rev": "5163432afc817cf8bd1f031418d1869e4c9d5547",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nix-community",
-        "repo": "nix-github-actions",
-        "type": "github"
-      }
-    },
     "nix-test-runner": {
       "flake": false,
       "locked": {
@@ -753,31 +714,6 @@
         "type": "github"
       }
     },
-    "poetry2nix": {
-      "inputs": {
-        "flake-utils": "flake-utils_7",
-        "nix-github-actions": "nix-github-actions",
-        "nixpkgs": [
-          "tgi-nix",
-          "nixpkgs"
-        ],
-        "systems": "systems_8",
-        "treefmt-nix": "treefmt-nix"
-      },
-      "locked": {
-        "lastModified": 1723854676,
-        "narHash": "sha256-+BrHfNuXrqeE7PoV6xDaoh0joYiJkvTTCIV0fFR3THw=",
-        "owner": "nix-community",
-        "repo": "poetry2nix",
-        "rev": "d650118bce34c0238b9b54f23f7f173f9e4db867",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nix-community",
-        "repo": "poetry2nix",
-        "type": "github"
-      }
-    },
     "pre-commit-hooks": {
       "inputs": {
         "flake-compat": [
@@ -887,7 +823,6 @@
           "tgi-nix",
           "nixpkgs"
         ],
-        "poetry2nix": "poetry2nix",
         "rust-overlay": "rust-overlay",
         "tgi-nix": "tgi-nix"
       }
@@ -1003,35 +938,6 @@
         "type": "github"
       }
     },
-    "systems_7": {
-      "locked": {
-        "lastModified": 1681028828,
-        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
-        "owner": "nix-systems",
-        "repo": "default",
-        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nix-systems",
-        "repo": "default",
-        "type": "github"
-      }
-    },
-    "systems_8": {
-      "locked": {
-        "lastModified": 1681028828,
-        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
-        "owner": "nix-systems",
-        "repo": "default",
-        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
-        "type": "github"
-      },
-      "original": {
-        "id": "systems",
-        "type": "indirect"
-      }
-    },
     "tgi-nix": {
       "inputs": {
         "flake-compat": "flake-compat_4",
@@ -1050,27 +956,6 @@
         "repo": "tgi-nix",
         "type": "github"
       }
-    },
-    "treefmt-nix": {
-      "inputs": {
-        "nixpkgs": [
-          "poetry2nix",
-          "nixpkgs"
-        ]
-      },
-      "locked": {
-        "lastModified": 1719749022,
-        "narHash": "sha256-ddPKHcqaKCIFSFc/cvxS14goUhCOAwsM1PbMr0ZtHMg=",
-        "owner": "numtide",
-        "repo": "treefmt-nix",
-        "rev": "8df5ff62195d4e67e2264df0b7f5e8c9995fd0bd",
-        "type": "github"
-      },
-      "original": {
-        "owner": "numtide",
-        "repo": "treefmt-nix",
-        "type": "github"
-      }
     }
   },
   "root": "root",
diff --git a/flake.nix b/flake.nix
index 299e6b3d..adc70fd1 100644
--- a/flake.nix
+++ b/flake.nix
@@ -8,10 +8,6 @@
     tgi-nix.url = "github:danieldk/tgi-nix";
     nixpkgs.follows = "tgi-nix/nixpkgs";
     flake-utils.url = "github:numtide/flake-utils";
-    poetry2nix = {
-      url = "github:nix-community/poetry2nix";
-      inputs.nixpkgs.follows = "tgi-nix/nixpkgs";
-    };
     rust-overlay = {
       url = "github:oxalica/rust-overlay";
       inputs.nixpkgs.follows = "tgi-nix/nixpkgs";
@@ -26,7 +22,6 @@
       flake-utils,
       rust-overlay,
       tgi-nix,
-      poetry2nix,
     }:
     flake-utils.lib.eachDefaultSystem (
       system:
@@ -47,14 +42,28 @@
             tgi-nix.overlay
           ];
         };
-        inherit (poetry2nix.lib.mkPoetry2Nix { inherit pkgs; }) mkPoetryEditablePackage;
-        text-generation-server = mkPoetryEditablePackage { editablePackageSources = ./server; };
         crateOverrides = import ./nix/crate-overrides.nix { inherit pkgs nix-filter; };
+        launcher = cargoNix.workspaceMembers.text-generation-launcher.build.override {
+          inherit crateOverrides;
+        };
+        router = cargoNix.workspaceMembers.text-generation-router-v3.build.override {
+          inherit crateOverrides;
+        };
+        server = pkgs.python3.pkgs.callPackage ./nix/server.nix { inherit nix-filter; };
       in
       {
-        devShells.default =
-          with pkgs;
-          mkShell {
+        devShells = with pkgs; rec {
+          default = pure;
+
+          pure = mkShell {
+            buildInputs = [
+              launcher
+              router
+              server
+            ];
+          };
+
+          impure = mkShell {
             buildInputs =
               [
                 openssl.dev
@@ -65,43 +74,16 @@
                     "rust-src"
                   ];
                 })
+                protobuf
               ]
               ++ (with python3.pkgs; [
                 venvShellHook
                 pip
-
-                causal-conv1d
-                click
-                einops
-                exllamav2
-                fbgemm-gpu
-                flashinfer
-                flash-attn
-                flash-attn-layer-norm
-                flash-attn-rotary
-                grpc-interceptor
-                grpcio-reflection
-                grpcio-status
-                grpcio-tools
-                hf-transfer
                 ipdb
-                loguru
-                mamba-ssm
-                marlin-kernels
-                opentelemetry-api
-                opentelemetry-exporter-otlp
-                opentelemetry-instrumentation-grpc
-                opentelemetry-semantic-conventions
-                peft
-                tokenizers
-                torch
-                transformers
-                vllm
-
-                (cargoNix.workspaceMembers.text-generation-launcher.build.override { inherit crateOverrides; })
-                (cargoNix.workspaceMembers.text-generation-router-v3.build.override { inherit crateOverrides; })
               ]);
 
+            inputsFrom = [ server ];
+
             venvDir = "./.venv";
 
             postVenv = ''
@@ -109,8 +91,10 @@
             '';
             postShellHook = ''
               unset SOURCE_DATE_EPOCH
+              export PATH=$PATH:~/.cargo/bin
             '';
           };
+        };
       }
     );
 }
diff --git a/nix/server.nix b/nix/server.nix
new file mode 100644
index 00000000..ff40757a
--- /dev/null
+++ b/nix/server.nix
@@ -0,0 +1,105 @@
+{
+  nix-filter,
+  buildPythonPackage,
+  poetry-core,
+  mypy-protobuf,
+  causal-conv1d,
+  einops,
+  exllamav2,
+  fbgemm-gpu,
+  flashinfer,
+  flash-attn,
+  flash-attn-layer-norm,
+  flash-attn-rotary,
+  grpc-interceptor,
+  grpcio-reflection,
+  grpcio-status,
+  grpcio-tools,
+  hf-transfer,
+  loguru,
+  mamba-ssm,
+  marlin-kernels,
+  opentelemetry-api,
+  opentelemetry-exporter-otlp,
+  opentelemetry-instrumentation-grpc,
+  opentelemetry-semantic-conventions,
+  peft,
+  safetensors,
+  tokenizers,
+  sentencepiece,
+  transformers,
+  typer,
+  vllm,
+}:
+
+let
+  filter = nix-filter.lib;
+in
+buildPythonPackage {
+  name = "text-generation-server";
+
+  src = filter {
+    root = ../.;
+    include = with filter; [
+      isDirectory
+      (and (inDirectory "server") (or_ (matchExt "py") (matchExt "pyi")))
+      "server/pyproject.toml"
+      (and (inDirectory "proto/v3") (matchExt "proto"))
+    ];
+  };
+
+  pyproject = true;
+
+  build-system = [ poetry-core ];
+
+  nativeBuildInputs = [ mypy-protobuf ];
+
+  pythonRelaxDeps = [
+    "einops"
+    "huggingface-hub"
+    "loguru"
+    "opentelemetry-instrumentation-grpc"
+    "sentencepiece"
+    "typer"
+  ];
+
+  pythonRemoveDeps = [ "scipy" ];
+
+  dependencies = [
+    causal-conv1d
+    einops
+    exllamav2
+    fbgemm-gpu
+    flashinfer
+    flash-attn
+    flash-attn-layer-norm
+    flash-attn-rotary
+    grpc-interceptor
+    grpcio-reflection
+    grpcio-status
+    grpcio-tools
+    hf-transfer
+    loguru
+    mamba-ssm
+    marlin-kernels
+    opentelemetry-api
+    opentelemetry-exporter-otlp
+    opentelemetry-instrumentation-grpc
+    opentelemetry-semantic-conventions
+    peft
+    safetensors
+    sentencepiece
+    tokenizers
+    transformers
+    typer
+    vllm
+  ];
+
+  prePatch = ''
+    python -m grpc_tools.protoc -Iproto/v3 --python_out=server/text_generation_server/pb \
+           --grpc_python_out=server/text_generation_server/pb --mypy_out=server/text_generation_server/pb proto/v3/generate.proto
+    find server/text_generation_server/pb/ -type f -name "*.py" -print0 -exec sed -i -e 's/^\(import.*pb2\)/from . \1/g' {} \;
+    touch server/text_generation_server/pb/__init__.py
+    cd server
+  '';
+}

From 947441509580c56e9bd8160bf2cf23c653d19abd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Dani=C3=ABl=20de=20Kok?= <me@danieldk.eu>
Date: Wed, 21 Aug 2024 07:48:13 +0200
Subject: [PATCH 59/72] nix: add `text-generation-benchmark` to pure devshell
 (#2431)

nix: add text-generation-benchmark to pure devshell
---
 flake.nix               |  4 +++
 nix/crate-overrides.nix | 68 ++++++++++++++++++++++++++---------------
 2 files changed, 47 insertions(+), 25 deletions(-)

diff --git a/flake.nix b/flake.nix
index adc70fd1..1c9c77a9 100644
--- a/flake.nix
+++ b/flake.nix
@@ -43,6 +43,9 @@
           ];
         };
         crateOverrides = import ./nix/crate-overrides.nix { inherit pkgs nix-filter; };
+        benchmark = cargoNix.workspaceMembers.text-generation-benchmark.build.override {
+          inherit crateOverrides;
+        };
         launcher = cargoNix.workspaceMembers.text-generation-launcher.build.override {
           inherit crateOverrides;
         };
@@ -57,6 +60,7 @@
 
           pure = mkShell {
             buildInputs = [
+              benchmark
               launcher
               router
               server
diff --git a/nix/crate-overrides.nix b/nix/crate-overrides.nix
index 343b3b25..cbf366af 100644
--- a/nix/crate-overrides.nix
+++ b/nix/crate-overrides.nix
@@ -20,34 +20,52 @@ defaultCrateOverrides
   rav1e = attrs: { env.CARGO_ENCODED_RUSTFLAGS = "-C target-feature=-crt-static"; };
 
   grpc-metadata = attrs: {
-    src =
-      filter {
-        root = ../backends/grpc-metadata;
-        include = with filter; [
-          isDirectory
-          (matchExt "rs")
-        ];
-      };
+    src = filter {
+      root = ../backends/grpc-metadata;
+      include = with filter; [
+        isDirectory
+        (matchExt "rs")
+      ];
+    };
   };
-  text-generation-launcer = attrs: {
-    src =
-      filter {
-        root = ../launcher;
-        include = with filter; [
-          isDirectory
-          (matchExt "rs")
-        ];
-      };
+  text-generation-benchmark = attrs: {
+    src = filter {
+      root = ../benchmark;
+      include = with filter; [
+        isDirectory
+        (matchExt "rs")
+      ];
+    };
+  };
+  text-generation-client = attrs: {
+    src = filter {
+      root = ../.;
+      include = with filter; [
+        isDirectory
+        (and (inDirectory "backends/client") (matchExt "rs"))
+        (and (inDirectory "proto") (matchExt "proto"))
+      ];
+    };
+    postPatch = "cd backends/client";
+    buildInputs = [ protobuf ];
+  };
+  text-generation-launcher = attrs: {
+    src = filter {
+      root = ../launcher;
+      include = with filter; [
+        isDirectory
+        (matchExt "rs")
+      ];
+    };
   };
   text-generation-router = attrs: {
-    src =
-      filter {
-        root = ../router;
-        include = with filter; [
-          isDirectory
-          (matchExt "rs")
-        ];
-      };
+    src = filter {
+      root = ../router;
+      include = with filter; [
+        isDirectory
+        (matchExt "rs")
+      ];
+    };
   };
   text-generation-router-v3 = attrs: {
     # We need to do the src/source root dance so that the build

From 310778e02a5aa36a4c72601e4e929bbbea0f1e7b Mon Sep 17 00:00:00 2001
From: Nicolas Patry <patry.nicolas@protonmail.com>
Date: Wed, 21 Aug 2024 09:06:33 +0200
Subject: [PATCH 60/72] Adding eetq to flake. (#2438)

---
 flake.lock     | 12 ++++++------
 nix/server.nix |  2 ++
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/flake.lock b/flake.lock
index c20dd98a..69f9ef13 100644
--- a/flake.lock
+++ b/flake.lock
@@ -835,11 +835,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1723602049,
-        "narHash": "sha256-Z/noCSn9WPkv7O77dWKLcBxe4Ub4bWyNzsL5JhjaQfw=",
+        "lastModified": 1724206841,
+        "narHash": "sha256-L8dKaX4T3k+TR2fEHCfGbH4UXdspovz/pj87iai9qmc=",
         "owner": "oxalica",
         "repo": "rust-overlay",
-        "rev": "ea0bf33a11a26a62c60123c49d96011da396602c",
+        "rev": "45e98fbd62c32e5927e952d2833fa1ba4fb35a61",
         "type": "github"
       },
       "original": {
@@ -944,11 +944,11 @@
         "nixpkgs": "nixpkgs_6"
       },
       "locked": {
-        "lastModified": 1723973328,
-        "narHash": "sha256-q5FmW4YFQcRb6fXHnrxL0uno6xcw9dcg+pFBbVM1xeQ=",
+        "lastModified": 1724218652,
+        "narHash": "sha256-Y7Kt+AZRIdo7tr/VhKGzdwYf7stiYQ4JD7flusEpXQw=",
         "owner": "danieldk",
         "repo": "tgi-nix",
-        "rev": "d2038f36589a8a179834e5771ffd081620ba94c3",
+        "rev": "ab2761aa7b970e737492b8cc41ca580dcb094808",
         "type": "github"
       },
       "original": {
diff --git a/nix/server.nix b/nix/server.nix
index ff40757a..1f90e3fd 100644
--- a/nix/server.nix
+++ b/nix/server.nix
@@ -4,6 +4,7 @@
   poetry-core,
   mypy-protobuf,
   causal-conv1d,
+  eetq,
   einops,
   exllamav2,
   fbgemm-gpu,
@@ -66,6 +67,7 @@ buildPythonPackage {
   pythonRemoveDeps = [ "scipy" ];
 
   dependencies = [
+    eetq
     causal-conv1d
     einops
     exllamav2

From 358ceb67dd343f4022537a1f28bc3fc9baec9102 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Dani=C3=ABl=20de=20Kok?= <me@danieldk.eu>
Date: Wed, 21 Aug 2024 22:20:03 +0200
Subject: [PATCH 61/72] nix: add awq-inference-engine as server dependency
 (#2442)

---
 flake.lock     | 6 +++---
 nix/server.nix | 2 ++
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/flake.lock b/flake.lock
index 69f9ef13..b40f51b3 100644
--- a/flake.lock
+++ b/flake.lock
@@ -944,11 +944,11 @@
         "nixpkgs": "nixpkgs_6"
       },
       "locked": {
-        "lastModified": 1724218652,
-        "narHash": "sha256-Y7Kt+AZRIdo7tr/VhKGzdwYf7stiYQ4JD7flusEpXQw=",
+        "lastModified": 1724270760,
+        "narHash": "sha256-KX566x0+3HZcB20HPdvdwyMm7ZJg21M+iqVrs/HCimA=",
         "owner": "danieldk",
         "repo": "tgi-nix",
-        "rev": "ab2761aa7b970e737492b8cc41ca580dcb094808",
+        "rev": "12cbaa76ff258351741d3b5afb7161f617fe7b4c",
         "type": "github"
       },
       "original": {
diff --git a/nix/server.nix b/nix/server.nix
index 1f90e3fd..4e0fdaa1 100644
--- a/nix/server.nix
+++ b/nix/server.nix
@@ -3,6 +3,7 @@
   buildPythonPackage,
   poetry-core,
   mypy-protobuf,
+  awq-inference-engine,
   causal-conv1d,
   eetq,
   einops,
@@ -67,6 +68,7 @@ buildPythonPackage {
   pythonRemoveDeps = [ "scipy" ];
 
   dependencies = [
+    awq-inference-engine
     eetq
     causal-conv1d
     einops

From f3c5d7d92f005c3cd6a801a33334fb9ba32f55f8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Dani=C3=ABl=20de=20Kok?= <me@danieldk.eu>
Date: Fri, 23 Aug 2024 22:06:22 +0200
Subject: [PATCH 62/72] nix: add default package (#2453)

The default package wraps the launcher and puts the server/router in the
path.

As a result, TGI can be started using something like:

```
nix run .# -- \
  --model-id hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4 \
  --port 8080
```
---
 flake.nix | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/flake.nix b/flake.nix
index 1c9c77a9..83feb26a 100644
--- a/flake.nix
+++ b/flake.nix
@@ -99,6 +99,17 @@
             '';
           };
         };
+
+        packages.default = pkgs.writeShellApplication {
+          name = "text-generation-inference";
+          runtimeInputs = [
+            server
+            router
+          ];
+          text = ''
+            ${launcher}/bin/text-generation-launcher "$@"
+          '';
+        };
       }
     );
 }

From 30be188400d27b6fedd88cb3dfd88de45639703c Mon Sep 17 00:00:00 2001
From: drbh <david.richard.holtz@gmail.com>
Date: Mon, 26 Aug 2024 17:04:46 -0400
Subject: [PATCH 63/72] Fix: don't apply post layernorm in
 SiglipVisionTransformer (#2459)

* Fix: don't apply post layernorm in SiglipVisionTransformer

This fixes a bug with LLaVA Next when using Siglip as the vision model. LLaVA Next expects the output of the vision model to be the encoder outputs before layernorm (see original transformers implementation here: https://github.com/huggingface/transformers/blob/main/src/transformers/models/llava_next/modeling_llava_next.py#L813).

This also makes Siglip consistent with the existing Clip implementation:

https://github.com/huggingface/text-generation-inference/blob/main/server/text_generation_server/models/custom_modeling/clip.py#L613

* fix: adjust pali gemma for post layer norm and small refactors

---------

Co-authored-by: Travis Addair <tgaddair@gmail.com>
---
 .../custom_modeling/flash_pali_gemma_modeling.py    | 10 +++++++++-
 .../models/custom_modeling/siglip.py                | 13 +------------
 2 files changed, 10 insertions(+), 13 deletions(-)

diff --git a/server/text_generation_server/models/custom_modeling/flash_pali_gemma_modeling.py b/server/text_generation_server/models/custom_modeling/flash_pali_gemma_modeling.py
index d10efb41..e08a2aad 100644
--- a/server/text_generation_server/models/custom_modeling/flash_pali_gemma_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/flash_pali_gemma_modeling.py
@@ -34,6 +34,11 @@ class PaliGemmaForConditionalGeneration(nn.Module):
             config=config.vision_config,
             weights=weights,
         )
+        self.post_vision_tower_layernorm = nn.LayerNorm.load(
+            prefix="vision_tower.vision_model.post_layernorm",
+            weights=weights,
+            eps=config.vision_config.layer_norm_eps,
+        )
 
         self.multi_modal_projector = TensorParallelColumnLinear.load(
             config,
@@ -84,7 +89,10 @@ class PaliGemmaForConditionalGeneration(nn.Module):
         if pixel_values is not None:
             pixel_values = pixel_values.to(dtype=inputs_embeds.dtype)
             image_outputs = self.vision_tower(pixel_values)
-            image_features = self.multi_modal_projector(image_outputs.last_hidden_state)
+            last_hidden_state = self.post_vision_tower_layernorm(
+                image_outputs.last_hidden_state
+            )
+            image_features = self.multi_modal_projector(last_hidden_state)
 
             # mask where image or padding tokens
             mask = input_ids == self.config.image_token_index
diff --git a/server/text_generation_server/models/custom_modeling/siglip.py b/server/text_generation_server/models/custom_modeling/siglip.py
index 480d0f9f..95ac9ede 100644
--- a/server/text_generation_server/models/custom_modeling/siglip.py
+++ b/server/text_generation_server/models/custom_modeling/siglip.py
@@ -364,7 +364,6 @@ class SiglipEncoder(nn.Module):
         inputs_embeds,
         attention_mask: Optional[torch.Tensor] = None,
     ):
-
         hidden_states = inputs_embeds
         for idx, encoder_layer in enumerate(self.layers):
             hidden_states, _ = encoder_layer(
@@ -386,20 +385,11 @@ class SiglipVisionTransformer(nn.Module):
         self.encoder = SiglipEncoder(
             prefix=f"{prefix}.encoder", config=config, weights=weights
         )
-        self.post_layernorm = nn.LayerNorm.load(
-            prefix=f"{prefix}.post_layernorm",
-            weights=weights,
-            eps=config.layer_norm_eps,
-        )
 
     def forward(
         self,
         pixel_values: Optional[torch.FloatTensor] = None,
     ):
-        r"""
-        Returns:
-
-        """
         if pixel_values is None:
             raise ValueError("You have to specify pixel_values")
 
@@ -412,10 +402,9 @@ class SiglipVisionTransformer(nn.Module):
             inputs_embeds=hidden_states,
         )
         last_hidden_state = encoder_outputs
-        post_last_hidden_state = self.post_layernorm(last_hidden_state)
 
         return BaseModelOutputWithPooling(
-            last_hidden_state=post_last_hidden_state,
+            last_hidden_state=last_hidden_state,
             # pooler_output=pooled_output,
             # hidden_states=encoder_outputs,
         )

From cfa73b5c99bc009903fbc340f8b77a6d4674455d Mon Sep 17 00:00:00 2001
From: drbh <david.richard.holtz@gmail.com>
Date: Mon, 26 Aug 2024 20:19:38 -0400
Subject: [PATCH 64/72] Pr 2451 ci branch (#2454)

* fix[router]: Fix tools not passed in chat template

Signed-off-by: GitHub <noreply@github.com>

* feat: improve default tool serialization and lints

* feat: refactor tool logic to include notify_error in prompt and adjust typing

* fix: adjust non tool template apply

* fix: simplify tool grammar logic and improve schema

* feat: avoid skip tool test and avoid empty tool prompts

* fix: increase test client timeout for grammar compilation tests

---------

Signed-off-by: GitHub <noreply@github.com>
Co-authored-by: Simone Rossi <simone.rossi.93@gmail.com>
---
 Cargo.lock                                   |   1 +
 clients/python/text_generation/client.py     |   7 +-
 docs/openapi.json                            |   2 +-
 integration-tests/conftest.py                |   2 +-
 integration-tests/models/test_tools_llama.py |  50 +++---
 router/Cargo.toml                            |   2 +-
 router/src/infer/chat_template.rs            |  57 ++++---
 router/src/infer/mod.rs                      |   6 +-
 router/src/infer/tool_grammar.rs             | 121 +++++++-------
 router/src/lib.rs                            |  15 +-
 router/src/server.rs                         | 160 ++++++++++++++++---
 11 files changed, 268 insertions(+), 155 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index d298c379..aa5cb642 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2174,6 +2174,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "45f7e8e35b6c7b169bf40b0176d2c79291ab8ee53290b84e0668ab21d841aa9d"
 dependencies = [
  "serde",
+ "serde_json",
 ]
 
 [[package]]
diff --git a/clients/python/text_generation/client.py b/clients/python/text_generation/client.py
index 12966747..45301b63 100644
--- a/clients/python/text_generation/client.py
+++ b/clients/python/text_generation/client.py
@@ -757,7 +757,12 @@ class AsyncClient:
                         continue
                     payload = byte_payload.decode("utf-8")
                     if payload.startswith("data:"):
-                        json_payload = json.loads(payload.lstrip("data:").rstrip("\n"))
+                        payload_data = (
+                            payload.lstrip("data:").rstrip("\n").removeprefix(" ")
+                        )
+                        if payload_data == "[DONE]":
+                            break
+                        json_payload = json.loads(payload_data)
                         try:
                             response = ChatCompletionChunk(**json_payload)
                             yield response
diff --git a/docs/openapi.json b/docs/openapi.json
index df21e19d..fd64a3ab 100644
--- a/docs/openapi.json
+++ b/docs/openapi.json
@@ -924,7 +924,7 @@
           "tool_prompt": {
             "type": "string",
             "description": "A prompt to be appended before the tools",
-            "example": "\"You will be presented with a JSON schema representing a set of tools.\nIf the user request lacks of sufficient information to make a precise tool selection: Do not invent any tool's properties, instead notify with an error message.\n\nJSON Schema:\n\"",
+            "example": "Given the functions available, please respond with a JSON for a function call with its proper arguments that best answers the given prompt. Respond in the format {name: function name, parameters: dictionary of argument name and its value}.Do not use variables.",
             "nullable": true
           },
           "tools": {
diff --git a/integration-tests/conftest.py b/integration-tests/conftest.py
index 15af1cad..a8a77cd2 100644
--- a/integration-tests/conftest.py
+++ b/integration-tests/conftest.py
@@ -257,7 +257,7 @@ class IgnoreLogProbResponseComparator(ResponseComparator):
 
 class LauncherHandle:
     def __init__(self, port: int):
-        self.client = AsyncClient(f"http://localhost:{port}")
+        self.client = AsyncClient(f"http://localhost:{port}", timeout=30)
 
     def _inner_health(self):
         raise NotImplementedError
diff --git a/integration-tests/models/test_tools_llama.py b/integration-tests/models/test_tools_llama.py
index f831990a..9855cfda 100644
--- a/integration-tests/models/test_tools_llama.py
+++ b/integration-tests/models/test_tools_llama.py
@@ -36,6 +36,7 @@ tools = [
                     },
                 },
                 "required": ["location", "format"],
+                "additionalProperties": False,
             },
         },
     },
@@ -62,13 +63,13 @@ tools = [
                     },
                 },
                 "required": ["location", "format", "num_days"],
+                "additionalProperties": False,
             },
         },
     },
 ]
 
 
-@pytest.mark.skip(reason="Takes too long to run")
 @pytest.mark.asyncio
 @pytest.mark.private
 async def test_flash_llama_grammar_tools(flash_llama_grammar_tools, response_snapshot):
@@ -76,7 +77,7 @@ async def test_flash_llama_grammar_tools(flash_llama_grammar_tools, response_sna
         max_tokens=100,
         seed=1,
         tools=tools,
-        presence_penalty=-1.1,
+        temperature=0.0,
         messages=[
             {
                 "role": "system",
@@ -91,19 +92,18 @@ async def test_flash_llama_grammar_tools(flash_llama_grammar_tools, response_sna
     assert response.choices[0].message.content is None
     assert response.choices[0].message.tool_calls == [
         {
-            "id": 0,
+            "id": "0",
             "type": "function",
             "function": {
                 "description": None,
                 "name": "get_current_weather",
-                "arguments": {"format": "celsius", "location": "New York, NY"},
+                "arguments": {"format": "celsius", "location": "Brooklyn, NY"},
             },
         }
     ]
     assert response == response_snapshot
 
 
-@pytest.mark.skip(reason="Takes too long to run")
 @pytest.mark.asyncio
 @pytest.mark.private
 async def test_flash_llama_grammar_tools_auto(
@@ -113,8 +113,8 @@ async def test_flash_llama_grammar_tools_auto(
         max_tokens=100,
         seed=1,
         tools=tools,
+        temperature=0.0,
         tool_choice="auto",
-        presence_penalty=-1.1,
         messages=[
             {
                 "role": "system",
@@ -129,12 +129,12 @@ async def test_flash_llama_grammar_tools_auto(
     assert response.choices[0].message.content is None
     assert response.choices[0].message.tool_calls == [
         {
-            "id": 0,
+            "id": "0",
             "type": "function",
             "function": {
                 "description": None,
                 "name": "get_current_weather",
-                "arguments": {"format": "celsius", "location": "New York, NY"},
+                "arguments": {"format": "celsius", "location": "Brooklyn, NY"},
             },
         }
     ]
@@ -142,7 +142,6 @@ async def test_flash_llama_grammar_tools_auto(
     assert response == response_snapshot
 
 
-@pytest.mark.skip(reason="Takes too long to run")
 @pytest.mark.asyncio
 @pytest.mark.private
 async def test_flash_llama_grammar_tools_choice(
@@ -152,8 +151,8 @@ async def test_flash_llama_grammar_tools_choice(
         max_tokens=100,
         seed=1,
         tools=tools,
+        temperature=0.0,
         tool_choice="get_current_weather",
-        presence_penalty=-1.1,
         messages=[
             {
                 "role": "system",
@@ -168,12 +167,12 @@ async def test_flash_llama_grammar_tools_choice(
     assert response.choices[0].message.content is None
     assert response.choices[0].message.tool_calls == [
         {
-            "id": 0,
+            "id": "0",
             "type": "function",
             "function": {
                 "description": None,
                 "name": "get_current_weather",
-                "arguments": {"format": "celsius", "location": "New York, NY"},
+                "arguments": {"format": "celsius", "location": "Brooklyn, NY"},
             },
         }
     ]
@@ -181,7 +180,6 @@ async def test_flash_llama_grammar_tools_choice(
     assert response == response_snapshot
 
 
-@pytest.mark.skip(reason="Takes too long to run")
 @pytest.mark.asyncio
 @pytest.mark.private
 async def test_flash_llama_grammar_tools_stream(
@@ -191,8 +189,8 @@ async def test_flash_llama_grammar_tools_stream(
         max_tokens=100,
         seed=1,
         tools=tools,
+        temperature=0.0,
         tool_choice="get_current_weather",
-        presence_penalty=-1.1,
         messages=[
             {
                 "role": "system",
@@ -210,11 +208,10 @@ async def test_flash_llama_grammar_tools_stream(
     async for response in responses:
         count += 1
 
-    assert count == 38
+    assert count == 48
     assert response == response_snapshot
 
 
-@pytest.mark.skip(reason="Takes too long to run")
 @pytest.mark.asyncio
 @pytest.mark.private
 async def test_flash_llama_grammar_tools_insufficient_information(
@@ -222,13 +219,13 @@ async def test_flash_llama_grammar_tools_insufficient_information(
 ):
     responses = await flash_llama_grammar_tools.chat(
         max_tokens=100,
-        seed=8,
+        seed=24,
         tools=tools,
         tool_choice="auto",
         messages=[
             {
                 "role": "system",
-                "content": "ONLY RESPOND IF THE USER ASKS A WEATHER RELATED QUESTION",
+                "content": "STRICTLY ONLY RESPOND IF THE USER ASKS A WEATHER RELATED QUESTION",
             },
             {
                 "role": "user",
@@ -239,18 +236,7 @@ async def test_flash_llama_grammar_tools_insufficient_information(
     )
 
     assert responses.choices[0].message.content is None
-    assert responses.choices[0].message.tool_calls == [
-        {
-            "function": {
-                "arguments": {
-                    "error": "Cannot get current weather forecast from specified location and temperature unit. Please try again with different options."
-                },
-                "description": None,
-                "name": "notify_error",
-            },
-            "id": 0,
-            "type": "function",
-        }
-    ]
-
+    assert (
+        responses.choices[0].message.tool_calls[0]["function"]["name"] == "notify_error"
+    )
     assert responses == response_snapshot
diff --git a/router/Cargo.toml b/router/Cargo.toml
index 7773e212..45acab8e 100644
--- a/router/Cargo.toml
+++ b/router/Cargo.toml
@@ -46,7 +46,7 @@ ngrok = { version = "0.13.1", features = ["axum"], optional = true }
 init-tracing-opentelemetry = { version = "0.14.1", features = [
   "opentelemetry-otlp",
 ] }
-minijinja = { version = "2.0.2" }
+minijinja = { version = "2.0.2", features = ["json"] }
 minijinja-contrib = { version = "2.0.2", features = ["pycompat"] }
 futures-util = "0.3.30"
 regex = "1.10.3"
diff --git a/router/src/infer/chat_template.rs b/router/src/infer/chat_template.rs
index a8537818..bfa9421c 100644
--- a/router/src/infer/chat_template.rs
+++ b/router/src/infer/chat_template.rs
@@ -1,9 +1,7 @@
 use std::collections::HashSet;
 
 use crate::infer::InferError;
-use crate::{
-    ChatTemplateInputs, GrammarType, Message, MessageChunk, TextMessage, TokenizerConfigToken,
-};
+use crate::{ChatTemplateInputs, Message, MessageChunk, TextMessage, TokenizerConfigToken, Tool};
 use minijinja::{Environment, ErrorKind, Template};
 use minijinja_contrib::pycompat;
 
@@ -32,6 +30,7 @@ impl ChatTemplate {
         env.set_unknown_method_callback(pycompat::unknown_method_callback);
         let template_str = template.into_boxed_str();
         env.add_function("raise_exception", raise_exception);
+        tracing::debug!("Loading template: {:#?}", template_str);
 
         // leaking env and template_str as read-only, static resources for performance.
         let template = Box::leak(env)
@@ -42,6 +41,7 @@ impl ChatTemplate {
         let variables = template.undeclared_variables(true);
         // check if the `tools` variable is used in the template
         let use_default_tool_template = !variables.contains("tools");
+        tracing::debug!("Use default tool template: {}", use_default_tool_template);
 
         Self {
             template,
@@ -56,25 +56,36 @@ impl ChatTemplate {
         &self,
         guideline: Option<&str>,
         mut messages: Vec<Message>,
-        grammar_with_prompt: Option<(GrammarType, String)>,
+        tools_and_prompt: Option<(Vec<Tool>, String)>,
     ) -> Result<String, InferError> {
-        if self.use_default_tool_template {
-            if let Some(last_message) = messages.last_mut() {
-                if let Some((GrammarType::Json(tools), tool_prompt)) = grammar_with_prompt {
-                    last_message.content.push(MessageChunk::Text {
-                        text: format!("\n---\n{}\n{}", tool_prompt, tools),
-                    });
-                }
-            }
-        }
-
-        let messages: Vec<TextMessage> = messages.into_iter().map(|c| c.into()).collect();
-
         // check if guideline is expected but not provided
         if self.variables.contains("guideline") && guideline.is_none() {
             return Err(InferError::MissingTemplateVariable("guideline".to_string()));
         }
 
+        let tools = match tools_and_prompt {
+            Some((tools, tool_prompt)) => {
+                // check if the `tools` variable is used in the template
+                // if not, we need to append the tools to the last message
+                let text = if self.use_default_tool_template {
+                    match serde_json::to_string(&tools) {
+                        Ok(tools_str) => format!("\n---\n{}\n{}", tools_str, tool_prompt),
+                        Err(e) => return Err(InferError::ToolError(e.to_string())),
+                    }
+                } else {
+                    // if the `tools` variable is used in the template, we just append the tool_prompt
+                    format!("\n---\n{}", tool_prompt)
+                };
+                if let Some(last_message) = messages.last_mut() {
+                    last_message.content.push(MessageChunk::Text { text });
+                }
+                Some(tools)
+            }
+            None => None,
+        };
+
+        let messages: Vec<TextMessage> = messages.into_iter().map(|c| c.into()).collect();
+
         self.template
             .render(ChatTemplateInputs {
                 guideline,
@@ -82,8 +93,7 @@ impl ChatTemplate {
                 bos_token: self.bos_token.as_deref(),
                 eos_token: self.eos_token.as_deref(),
                 add_generation_prompt: true,
-                tools: None,
-                tools_prompt: None,
+                tools,
             })
             .map_err(InferError::TemplateError)
     }
@@ -95,7 +105,7 @@ mod tests {
     use crate::infer::chat_template::raise_exception;
     use crate::infer::ChatTemplate;
     use crate::{
-        ChatTemplateInputs, GrammarType, Message, MessageContent, TextMessage, TokenizerConfigToken,
+        ChatTemplateInputs, Message, MessageContent, TextMessage, TokenizerConfigToken, Tool,
     };
     use minijinja::Environment;
 
@@ -854,11 +864,12 @@ mod tests {
                 content: MessageContent::SingleText("Just testing".to_string()),
             },
         ];
-        let tools = serde_json::json!("[]");
+        let tools_string = r#"[{"type": "function","function": {"name": "get_current_weather","description": "Get the current weather","parameters": {"type": "object","properties": {"location": {"type": "string","description": "The city and state, e.g. San Francisco, CA"},"format": {"type": "string","enum": ["celsius", "fahrenheit"],"description": "The temperature unit to use. Infer this from the users location."}},"required": ["location", "format"]}}}]"#.to_string();
+        let tools: Vec<Tool> = serde_json::from_str(&tools_string).unwrap();
         let tool_prompt = "This default prompt will be used".to_string();
-        let grammer_with_prompt = (GrammarType::Json(tools), tool_prompt);
-        let result = ct.apply(None, msgs, Some(grammer_with_prompt));
-        let expected = "<s>[INST] I'd like to show off how chat templating works! [/INST]Great! How can I help you today?</s> [INST] Just testing\n---\nThis default prompt will be used\n\"[]\" [/INST]".to_string();
+        let tools_and_prompt = Some((tools, tool_prompt));
+        let result = ct.apply(None, msgs, tools_and_prompt);
+        let expected = "<s>[INST] I'd like to show off how chat templating works! [/INST]Great! How can I help you today?</s> [INST] Just testing\n---\n[{\"type\":\"function\",\"function\":{\"description\":\"Get the current weather\",\"name\":\"get_current_weather\",\"arguments\":{\"properties\":{\"format\":{\"description\":\"The temperature unit to use. Infer this from the users location.\",\"enum\":[\"celsius\",\"fahrenheit\"],\"type\":\"string\"},\"location\":{\"description\":\"The city and state, e.g. San Francisco, CA\",\"type\":\"string\"}},\"required\":[\"location\",\"format\"],\"type\":\"object\"}}}]\nThis default prompt will be used [/INST]".to_string();
         assert_eq!(result.unwrap(), expected);
     }
 }
diff --git a/router/src/infer/mod.rs b/router/src/infer/mod.rs
index c9354d9a..81c0d38f 100644
--- a/router/src/infer/mod.rs
+++ b/router/src/infer/mod.rs
@@ -3,7 +3,7 @@ mod chat_template;
 pub mod tool_grammar;
 
 use crate::validation::{ValidGenerateRequest, Validation, ValidationError};
-use crate::GrammarType;
+use crate::Tool;
 use crate::{
     ChatTemplateVersions, FinishReason, GenerateRequest, HubProcessorConfig, HubTokenizerConfig,
     Message, PrefillToken, Token,
@@ -140,12 +140,12 @@ impl Infer {
         &self,
         guideline: Option<String>,
         messages: Vec<Message>,
-        grammar_with_prompt: Option<(GrammarType, String)>,
+        tools_and_prompt: Option<(Vec<Tool>, String)>,
     ) -> Result<String, InferError> {
         self.chat_template
             .as_ref()
             .ok_or_else(|| InferError::TemplateError(ErrorKind::TemplateNotFound.into()))?
-            .apply(guideline.as_deref(), messages, grammar_with_prompt)
+            .apply(guideline.as_deref(), messages, tools_and_prompt)
             .map_err(|e| {
                 metrics::counter!("tgi_request_failure", "err" => "template").increment(1);
                 tracing::error!("{e}");
diff --git a/router/src/infer/tool_grammar.rs b/router/src/infer/tool_grammar.rs
index 05027f30..4fe15720 100644
--- a/router/src/infer/tool_grammar.rs
+++ b/router/src/infer/tool_grammar.rs
@@ -1,5 +1,8 @@
 use crate::infer::InferError;
-use crate::{FunctionRef, FunctionsMap, Properties, Tool, ToolChoice, ToolType, Tools};
+use crate::{
+    FunctionDefinition, FunctionRef, FunctionsMap, JsonSchemaTool, Properties, Tool, ToolChoice,
+    ToolType,
+};
 use serde_json::{json, Map, Value};
 use std::collections::HashMap;
 
@@ -16,17 +19,38 @@ impl ToolGrammar {
     }
 
     pub fn apply(
-        tools: Option<Vec<Tool>>,
+        tools: Vec<Tool>,
         tool_choice: ToolChoice,
-    ) -> Result<Option<Tools>, InferError> {
+    ) -> Result<(Vec<Tool>, Option<JsonSchemaTool>), InferError> {
         // if no tools are provided, we return None
-        let tools = match tools {
-            Some(tools) if !tools.is_empty() => tools,
-            _ => return Ok(None),
-        };
+        if tools.is_empty() {
+            return Ok((tools, None));
+        }
 
         let tool_choice = tool_choice.0.unwrap_or(ToolType::OneOf);
 
+        let mut tools = tools.clone();
+
+        // add the notify_error function to the tools
+        let notify_error = Tool {
+            r#type: "function".to_string(),
+            function: FunctionDefinition {
+                name: "notify_error".to_string(),
+                description: Some("Notify an error or issue".to_string()),
+                arguments: json!({
+                    "type": "object",
+                    "properties": {
+                        "error": {
+                            "type": "string",
+                            "description": "The error or issue to notify"
+                        }
+                    },
+                    "required": ["error"]
+                }),
+            },
+        };
+        tools.push(notify_error);
+
         // if tools are provided and no tool_choice we default to the OneOf
         let tools_to_use = match tool_choice {
             ToolType::FunctionName(name) => {
@@ -35,87 +59,57 @@ impl ToolGrammar {
             ToolType::Function { function } => {
                 vec![Self::find_tool_by_name(&tools, &function.name)?]
             }
-            ToolType::OneOf => tools,
-            ToolType::NoTool => return Ok(None),
+            ToolType::OneOf => tools.clone(),
+            ToolType::NoTool => return Ok((tools, None)),
         };
 
-        // adds the error notification function for LLM feedback if required
-        let mut text_response_properties = Map::new();
-        text_response_properties.insert(
-            "error".to_string(),
-            serde_json::json!({
-                "type": "string",
-                "description": "The error or issue to notify"
-            }),
-        );
-        text_response_properties.insert(
-            "_name".to_string(),
-            serde_json::json!({
-                "type": "string",
-                "const": "notify_error"
-            }),
-        );
-
         let functions: HashMap<String, serde_json::Value> = tools_to_use
             .iter()
             .map(|tool| {
                 let func = tool.function.clone();
 
-                // Clone the existing parameters, which are expected to be a JSON object
-                let mut params = if let Value::Object(params) = &func.arguments {
-                    params.clone()
-                } else {
-                    Map::new()
-                };
+                let mut params = Map::new();
 
-                // Insert the function's description at the top level, outside of properties
                 params.insert(
                     "description".to_string(),
-                    Value::String(func.description.clone().unwrap_or_default()),
+                    Value::String(func.description.unwrap_or_default()),
                 );
 
-                // Ensure 'properties' exists and is an object
-                let properties = params
-                    .entry("properties".to_string())
-                    .or_insert_with(|| json!({}))
-                    .as_object_mut()
-                    .unwrap();
+                let mut properties = Map::new();
+                let mut required = vec![Value::String("_name".to_string())];
 
-                // Insert the constant for the function name inside 'properties'
                 properties.insert(
                     "_name".to_string(),
                     json!({
                         "type": "string",
                         "const": func.name.clone(),
-                        // "description": "The name of the function"
                     }),
                 );
 
-                // Check if 'required' exists, and it is an array. If not, create an empty array.
-                let required = params
-                    .entry("required".to_string())
-                    .or_insert_with(|| json!([]))
-                    .as_array_mut()
-                    .unwrap();
-
-                // Add 'name' to the 'required' array if it is not already present
-                if !required.iter().any(|r| r == "_name") {
-                    required.push(json!("_name"));
+                if let Value::Object(args) = func.arguments {
+                    if let Some(Value::Object(props)) = args.get("properties") {
+                        properties.extend(props.clone());
+                    }
+                    if let Some(Value::Array(reqs)) = args.get("required") {
+                        required.extend(reqs.clone());
+                    }
+                    params.insert(
+                        "additionalProperties".to_string(),
+                        Value::Bool(
+                            args.get("additionalProperties").and_then(|v| v.as_str())
+                                == Some("true"),
+                        ),
+                    );
                 }
 
+                params.insert("properties".to_string(), Value::Object(properties));
+                params.insert("required".to_string(), Value::Array(required));
+
                 (func.name, Value::Object(params))
             })
-            .chain([(
-                "notify_error".to_string(),
-                serde_json::json!({
-                    "properties": text_response_properties,
-                    "required": ["error", "_name"],
-                    "type": "object"
-                }),
-            )])
             .collect();
 
-        let tools = Tools {
+        let tool_schema = JsonSchemaTool {
             functions_map: FunctionsMap { functions },
             properties: Properties {
                 function: tools_to_use
@@ -123,13 +117,10 @@ impl ToolGrammar {
                     .map(|tool| FunctionRef {
                         ref_path: format!("#/$functions/{}", tool.function.name.clone()),
                     })
-                    .chain(std::iter::once(FunctionRef {
-                        ref_path: "#/$functions/notify_error".to_string(),
-                    }))
                     .collect(),
             },
         };
 
-        Ok(Some(tools))
+        Ok((tools, Some(tool_schema)))
     }
 }
diff --git a/router/src/lib.rs b/router/src/lib.rs
index 1b2ff153..ce4f7c46 100644
--- a/router/src/lib.rs
+++ b/router/src/lib.rs
@@ -840,10 +840,10 @@ pub(crate) struct ChatRequest {
     pub tools: Option<Vec<Tool>>,
 
     /// A prompt to be appended before the tools
-    #[serde(default = "default_tool_prompt")]
+    #[serde(default)]
     #[schema(
         nullable = true,
-        example = "\"You will be presented with a JSON schema representing a set of tools.\nIf the user request lacks of sufficient information to make a precise tool selection: Do not invent any tool's properties, instead notify with an error message.\n\nJSON Schema:\n\""
+        example = "Given the functions available, please respond with a JSON for a function call with its proper arguments that best answers the given prompt. Respond in the format {name: function name, parameters: dictionary of argument name and its value}.Do not use variables."
     )]
     pub tool_prompt: Option<String>,
 
@@ -865,10 +865,8 @@ pub(crate) struct ChatRequest {
     pub guideline: Option<String>,
 }
 
-fn default_tool_prompt() -> Option<String> {
-    Some(
-        "\nYou will be presented with a JSON schema representing a set of tools.\nIf the user request lacks of sufficient information to make a precise tool selection: Do not invent any tool's properties, instead notify with an error message.\n\nJSON Schema:\n".to_string(),
-    )
+pub fn default_tool_prompt() -> String {
+    "\nGiven the functions available, please respond with a JSON for a function call with its proper arguments that best answers the given prompt. Respond in the format {name: function name, parameters: dictionary of argument name and its value}.Do not use variables.\n".to_string()
 }
 
 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize, ToSchema)]
@@ -910,7 +908,7 @@ impl From<ToolTypeDeserializer> for ToolChoice {
 }
 
 #[derive(Debug, Deserialize, Serialize, ToSchema, PartialEq)]
-pub struct Tools {
+pub struct JsonSchemaTool {
     #[serde(flatten)]
     functions_map: FunctionsMap,
     properties: Properties,
@@ -968,8 +966,7 @@ pub(crate) struct ChatTemplateInputs<'a> {
     bos_token: Option<&'a str>,
     eos_token: Option<&'a str>,
     add_generation_prompt: bool,
-    tools: Option<&'a str>,
-    tools_prompt: Option<&'a str>,
+    tools: Option<Vec<Tool>>,
     guideline: Option<&'a str>,
 }
 
diff --git a/router/src/server.rs b/router/src/server.rs
index 8ec7a871..8ebd1a33 100644
--- a/router/src/server.rs
+++ b/router/src/server.rs
@@ -8,7 +8,7 @@ use crate::kserve::{
     kserve_model_metadata, kserve_model_metadata_ready,
 };
 use crate::validation::ValidationError;
-use crate::ChatTokenizeResponse;
+use crate::{default_tool_prompt, ChatTokenizeResponse};
 use crate::{
     usage_stats, BestOfSequence, Details, ErrorResponse, FinishReason, FunctionName,
     GenerateParameters, GenerateRequest, GenerateResponse, GrammarType, HubModelInfo,
@@ -23,7 +23,7 @@ use crate::{
     CompletionRequest, CompletionType, DeltaToolCall, Function, Prompt, Tool, VertexRequest,
     VertexResponse,
 };
-use crate::{FunctionDefinition, HubPreprocessorConfig, ToolCall, ToolChoice, ToolType, Tools};
+use crate::{FunctionDefinition, HubPreprocessorConfig, ToolCall, ToolChoice, ToolType};
 use async_stream::__private::AsyncStream;
 use axum::extract::Extension;
 use axum::http::{HeaderMap, HeaderValue, Method, StatusCode};
@@ -146,7 +146,7 @@ async fn get_chat_tokenize(
     } = req;
 
     let tool_prompt = tool_prompt.unwrap_or_default();
-    let (inputs, _grammar, _tool_grammar) = prepare_chat_input(
+    let (inputs, _grammar, _using_tools) = prepare_chat_input(
         &infer,
         response_format,
         tools,
@@ -1158,14 +1158,16 @@ async fn chat_completions(
     let repetition_penalty = presence_penalty.map(|x| x + 2.0);
     let max_new_tokens = max_tokens.or(Some(100));
     let logprobs = logprobs.unwrap_or(false);
-    let tool_prompt = tool_prompt.unwrap_or_default();
+    let tool_prompt = tool_prompt
+        .filter(|s| !s.is_empty())
+        .unwrap_or_else(default_tool_prompt);
     let stop = stop.unwrap_or_default();
     // enable greedy only when temperature is 0
     let (do_sample, temperature) = match temperature {
         Some(temperature) if temperature == 0.0 => (false, None),
         other => (true, other),
     };
-    let (inputs, grammar, tool_grammar) = prepare_chat_input(
+    let (inputs, grammar, using_tools) = prepare_chat_input(
         &infer,
         response_format,
         tools,
@@ -1221,7 +1223,7 @@ async fn chat_completions(
             });
 
             // replace the content with the tool calls if grammar is present
-            let (content, tool_calls) = if tool_grammar.is_some() {
+            let (content, tool_calls) = if using_tools {
                 (None, Some(vec![stream_token.token.text]))
             } else {
                 let content = if !stream_token.token.special {
@@ -1275,7 +1277,7 @@ async fn chat_completions(
             .unwrap_or_else(|_| std::time::Duration::from_secs(0))
             .as_secs();
 
-        let (tool_calls, output) = if tool_grammar.is_some() {
+        let (tool_calls, output) = if using_tools {
             let gen_text_value: Value =
                 serde_json::from_str(&generation.generated_text).map_err(|e| {
                     InferError::ToolError(format!(
@@ -2539,7 +2541,7 @@ fn create_post_processor(
     Ok(post_processor)
 }
 
-type PreparedInput = (String, Option<GrammarType>, Option<Tools>);
+type PreparedInput = (String, Option<GrammarType>, bool);
 
 fn prepare_chat_input(
     infer: &Infer,
@@ -2556,19 +2558,139 @@ fn prepare_chat_input(
         ));
     }
 
+    // when response_format is set, tools are not included when applying the chat template to generate inputs
     if let Some(format) = response_format {
         let inputs = infer.apply_chat_template(guideline, messages, None)?;
-        return Ok((inputs, Some(format), None));
+        return Ok((inputs, Some(format), false));
     }
 
-    // if tools are set, apply the tool grammar and then the chat template
-    let tool_grammar: Option<Tools> = ToolGrammar::apply(tools, tool_choice)?;
-    let grammar = tool_grammar
-        .as_ref()
-        .map(|t| GrammarType::Json(serde_json::json!(t)));
-    let tools_grammar_prompt = tool_grammar
-        .as_ref()
-        .map(|t| (GrammarType::Json(serde_json::json!(t)), tool_prompt.into()));
-    let inputs = infer.apply_chat_template(guideline, messages, tools_grammar_prompt)?;
-    Ok((inputs, grammar, tool_grammar))
+    // when no response_format is set and tools are included, apply the chat template with the tools
+    // to generate inputs
+    if let Some(tools) = tools {
+        let (updated_tools, tool_schema) = ToolGrammar::apply(tools, tool_choice)?;
+
+        let grammar = tool_schema
+            .as_ref()
+            .map(|t| GrammarType::Json(serde_json::json!(t)));
+
+        let inputs: String = infer.apply_chat_template(
+            guideline,
+            messages,
+            Some((updated_tools, tool_prompt.into())),
+        )?;
+        return Ok((inputs, grammar, tool_schema.is_some()));
+    }
+
+    // if no response_format or tools are set simply apply the chat template to generate inputs
+    let inputs = infer.apply_chat_template(guideline, messages, None)?;
+    Ok((inputs, None, false))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::ChatTemplateVersions;
+    use crate::HubTokenizerConfig;
+    use crate::TokenizerConfigToken;
+    use crate::Tool;
+
+    use serde_json::json;
+
+    #[test]
+    fn test_prepare_chat_input() {
+        // Mock Backend to avoid network requests
+        struct MockBackend;
+
+        impl Backend for MockBackend {
+            fn schedule(
+                &self,
+                _request: crate::validation::ValidGenerateRequest,
+            ) -> Result<
+                tokio_stream::wrappers::UnboundedReceiverStream<
+                    Result<InferStreamResponse, InferError>,
+                >,
+                InferError,
+            > {
+                unimplemented!("Never called in this test");
+            }
+            fn health<'a, 'async_trait>(
+                &'a self,
+                _current_health: bool,
+            ) -> core::pin::Pin<
+                Box<dyn core::future::Future<Output = bool> + core::marker::Send + 'async_trait>,
+            >
+            where
+                'a: 'async_trait,
+                Self: 'async_trait,
+            {
+                unimplemented!("Never called in this test");
+            }
+        }
+
+        let backend = MockBackend {};
+
+        let mut tokenizer_config = HubTokenizerConfig::default();
+
+        // mock tokenizer config values
+        tokenizer_config.bos_token = Some(TokenizerConfigToken::String("<s>".to_string()));
+        tokenizer_config.eos_token = Some(TokenizerConfigToken::String("</s>".to_string()));
+        tokenizer_config.chat_template = Some(
+            ChatTemplateVersions::Single("{%- if messages[0][\"role\"] == \"system\" %}\n    {%- set system_message = messages[0][\"content\"] %}\n    {%- set loop_messages = messages[1:] %}\n{%- else %}\n    {%- set loop_messages = messages %}\n{%- endif %}\n{%- if not tools is defined %}\n    {%- set tools = none %}\n{%- endif %}\n{%- set user_messages = loop_messages | selectattr(\"role\", \"equalto\", \"user\") | list %}\n\n{#- This block checks for alternating user/assistant messages, skipping tool calling messages #}\n{%- set ns = namespace() %}\n{%- set ns.index = 0 %}\n{%- for message in loop_messages %}\n    {%- if not (message.role == \"tool\" or message.role == \"tool_results\" or (message.tool_calls is defined and message.tool_calls is not none)) %}\n        {%- if (message[\"role\"] == \"user\") != (ns.index % 2 == 0) %}\n            {{- raise_exception(\"After the optional system message, conversation roles must alternate user/assistant/user/assistant/...\") }}\n        {%- endif %}\n        {%- set ns.index = ns.index + 1 %}\n    {%- endif %}\n{%- endfor %}\n\n{{- bos_token }}\n{%- for message in loop_messages %}\n    {%- if message[\"role\"] == \"user\" %}\n        {%- if tools is not none and (message == user_messages[-1]) %}\n            {{- \"[AVAILABLE_TOOLS] [\" }}\n            {%- for tool in tools %}\n                {%- set tool = tool.function %}\n                {{- '{\"type\": \"function\", \"function\": {' }}\n                {%- for key, val in tool.items() if key != \"return\" %}\n                    {%- if val is string %}\n                        {{- '\"' + key + '\": \"' + val + '\"' }}\n                    {%- else %}\n                        {{- '\"' + key + '\": ' + val|tojson }}\n                    {%- endif %}\n                    {%- if not loop.last %}\n                        {{- \", \" }}\n                    {%- endif %}\n                {%- endfor %}\n                {{- \"}}\" }}\n                {%- if not loop.last %}\n                    {{- \", \" }}\n                {%- else %}\n                    {{- \"]\" }}\n                {%- endif %}\n            {%- endfor %}\n            {{- \"[/AVAILABLE_TOOLS]\" }}\n            {%- endif %}\n        {%- if loop.last and system_message is defined %}\n            {{- \"[INST] \" + system_message + \"\\n\\n\" + message[\"content\"] + \"[/INST]\" }}\n        {%- else %}\n            {{- \"[INST] \" + message[\"content\"] + \"[/INST]\" }}\n        {%- endif %}\n    {%- elif message.tool_calls is defined and message.tool_calls is not none %}\n        {{- \"[TOOL_CALLS] [\" }}\n        {%- for tool_call in message.tool_calls %}\n            {%- set out = tool_call.function|tojson %}\n            {{- out[:-1] }}\n            {%- if not tool_call.id is defined or tool_call.id|length != 9 %}\n                {{- raise_exception(\"Tool call IDs should be alphanumeric strings with length 9!\") }}\n            {%- endif %}\n            {{- ', \"id\": \"' + tool_call.id + '\"}' }}\n            {%- if not loop.last %}\n                {{- \", \" }}\n            {%- else %}\n                {{- \"]\" + eos_token }}\n            {%- endif %}\n        {%- endfor %}\n    {%- elif message[\"role\"] == \"assistant\" %}\n        {{- \" \" + message[\"content\"]|trim + eos_token}}\n    {%- elif message[\"role\"] == \"tool_results\" or message[\"role\"] == \"tool\" %}\n        {%- if message.content is defined and message.content.content is defined %}\n            {%- set content = message.content.content %}\n        {%- else %}\n            {%- set content = message.content %}\n        {%- endif %}\n        {{- '[TOOL_RESULTS] {\"content\": ' + content|string + \", \" }}\n        {%- if not message.tool_call_id is defined or message.tool_call_id|length != 9 %}\n            {{- raise_exception(\"Tool call IDs should be alphanumeric strings with length 9!\") }}\n        {%- endif %}\n        {{- '\"call_id\": \"' + message.tool_call_id + '\"}[/TOOL_RESULTS]' }}\n    {%- else %}\n        {{- raise_exception(\"Only user and assistant roles are supported, with the exception of an initial optional system message!\") }}\n    {%- endif %}\n{%- endfor %}\n".to_string())
+        );
+
+        let infer = Infer::new(
+            backend,
+            Validation::new(1, None, None, None, 1, 1, 1, 1, 1, false),
+            1,
+            tokenizer_config,
+            HubProcessorConfig::default(),
+        );
+        let response_format = None;
+        let tools = Some(vec![Tool {
+            r#type: "function".to_string(),
+            function: FunctionDefinition {
+                name: "get_current_weather".to_string(),
+                description: Some("Get the current weather".to_string()),
+                arguments: json!({
+                    "type": "object",
+                    "properties": {
+                        "location": {
+                            "type": "string",
+                            "description": "The city and state, e.g. San Francisco, CA"
+                        },
+                        "format": {
+                            "type": "string",
+                            "enum": ["celsius", "fahrenheit"],
+                            "description": "The temperature unit to use. Infer this from the users location."
+                        }
+                    },
+                    "required": ["location", "format"]
+                }),
+            },
+        }]);
+        let tool_prompt = "Given the functions available, please respond with a JSON for a function call with its proper arguments that best answers the given prompt. Respond in the format {name: function name, parameters: dictionary of argument name and its value}.Do not use variables.";
+        let guideline = None;
+        let messages = vec![Message {
+            name: None,
+            role: "user".to_string(),
+            content: MessageContent::SingleText(
+                "What is the weather like in New York?".to_string(),
+            ),
+        }];
+
+        let result = prepare_chat_input(
+            &infer,
+            response_format,
+            tools,
+            ToolChoice(None),
+            tool_prompt,
+            guideline,
+            messages,
+        );
+
+        assert!(result.is_ok());
+        let (inputs, _grammar, using_tools) = result.unwrap();
+        assert_eq!(using_tools, true);
+        assert_eq!(inputs, "<s>[AVAILABLE_TOOLS] [{\"type\": \"function\", \"function\": {\"arguments\": {\"properties\":{\"format\":{\"description\":\"The temperature unit to use. Infer this from the users location.\",\"enum\":[\"celsius\",\"fahrenheit\"],\"type\":\"string\"},\"location\":{\"description\":\"The city and state, e.g. San Francisco, CA\",\"type\":\"string\"}},\"required\":[\"location\",\"format\"],\"type\":\"object\"}, \"description\": \"Get the current weather\", \"name\": \"get_current_weather\"}}, {\"type\": \"function\", \"function\": {\"arguments\": {\"properties\":{\"error\":{\"description\":\"The error or issue to notify\",\"type\":\"string\"}},\"required\":[\"error\"],\"type\":\"object\"}, \"description\": \"Notify an error or issue\", \"name\": \"notify_error\"}}][/AVAILABLE_TOOLS][INST] What is the weather like in New York?\n---\nGiven the functions available, please respond with a JSON for a function call with its proper arguments that best answers the given prompt. Respond in the format {name: function name, parameters: dictionary of argument name and its value}.Do not use variables.[/INST]".to_string());
+    }
 }

From 2788d41a76c193d4de7055dc5ef38a97f25c38b5 Mon Sep 17 00:00:00 2001
From: Nicolas Patry <patry.nicolas@protonmail.com>
Date: Tue, 27 Aug 2024 15:33:02 +0200
Subject: [PATCH 65/72] Fixing CI. (#2462)

---
 .github/workflows/build.yaml    | 4 ----
 .github/workflows/ci_build.yaml | 3 +++
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
index 1f72c46d..d415f369 100644
--- a/.github/workflows/build.yaml
+++ b/.github/workflows/build.yaml
@@ -32,10 +32,6 @@ jobs:
     permissions:
       contents: write
       packages: write
-      # This is used to complete the identity challenge
-      # with sigstore/fulcio when running outside of PRs.
-      id-token: write
-      security-events: write
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
diff --git a/.github/workflows/ci_build.yaml b/.github/workflows/ci_build.yaml
index 6000cec3..5190f321 100644
--- a/.github/workflows/ci_build.yaml
+++ b/.github/workflows/ci_build.yaml
@@ -39,6 +39,9 @@ jobs:
       matrix:
         hardware: ["cuda", "rocm", "intel-xpu", "intel-cpu"]
     uses: ./.github/workflows/build.yaml # calls the one above ^
+    permissions:
+      contents: write
+      packages: write
     with:
       hardware: ${{ matrix.hardware }}
       # https://github.com/actions/runner/issues/2206

From 21187c27c90acbec7f912b8af4feaec154de960f Mon Sep 17 00:00:00 2001
From: drbh <david.richard.holtz@gmail.com>
Date: Tue, 27 Aug 2024 13:31:08 -0400
Subject: [PATCH 66/72] fix: bump minijinja version and add test for llama 3.1
 tools (#2463)

* fix: support tojson and avoid message indexing issue in template

* fix: prefer minijinja native methods and prefer workspace level dependency

* fix: adjust comment typo
---
 Cargo.lock                        |  4 ++--
 Cargo.toml                        |  2 ++
 backends/v3/Cargo.toml            |  4 ++--
 router/Cargo.toml                 |  4 ++--
 router/src/infer/chat_template.rs | 37 +++++++++++++++++++++++++++++--
 5 files changed, 43 insertions(+), 8 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index aa5cb642..02e91bc1 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2169,9 +2169,9 @@ dependencies = [
 
 [[package]]
 name = "minijinja"
-version = "2.1.0"
+version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "45f7e8e35b6c7b169bf40b0176d2c79291ab8ee53290b84e0668ab21d841aa9d"
+checksum = "6d7d3e3a3eece1fa4618237ad41e1de855ced47eab705cec1c9a920e1d1c5aad"
 dependencies = [
  "serde",
  "serde_json",
diff --git a/Cargo.toml b/Cargo.toml
index 8bf75b90..79fda15d 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -29,6 +29,8 @@ tokenizers = { version = "0.19.1", features = ["http"] }
 hf-hub = { version = "0.3.1", features = ["tokio"] }
 metrics = { version = "0.23.0" }
 metrics-exporter-prometheus = { version = "0.15.1", features = [] }
+minijinja = { version = "2.2.0", features = ["json"] }
+minijinja-contrib = { version = "2.0.2", features = ["pycompat"] }
 
 [profile.release]
 incremental = true
diff --git a/backends/v3/Cargo.toml b/backends/v3/Cargo.toml
index 06a44bec..69dad072 100644
--- a/backends/v3/Cargo.toml
+++ b/backends/v3/Cargo.toml
@@ -53,8 +53,8 @@ utoipa-swagger-ui = { version = "6.0.0", features = ["axum"] }
 init-tracing-opentelemetry = { version = "0.14.1", features = [
   "opentelemetry-otlp",
 ] }
-minijinja = { version = "2.0.2" }
-minijinja-contrib = { version = "2.0.2", features = ["pycompat"] }
+minijinja = { workspace = true }
+minijinja-contrib = { workspace = true }
 futures-util = "0.3.30"
 regex = "1.10.3"
 once_cell = "1.19.0"
diff --git a/router/Cargo.toml b/router/Cargo.toml
index 45acab8e..5c328e8a 100644
--- a/router/Cargo.toml
+++ b/router/Cargo.toml
@@ -46,8 +46,8 @@ ngrok = { version = "0.13.1", features = ["axum"], optional = true }
 init-tracing-opentelemetry = { version = "0.14.1", features = [
   "opentelemetry-otlp",
 ] }
-minijinja = { version = "2.0.2", features = ["json"] }
-minijinja-contrib = { version = "2.0.2", features = ["pycompat"] }
+minijinja = { workspace = true }
+minijinja-contrib = { workspace = true }
 futures-util = "0.3.30"
 regex = "1.10.3"
 once_cell = "1.19.0"
diff --git a/router/src/infer/chat_template.rs b/router/src/infer/chat_template.rs
index bfa9421c..a736fc12 100644
--- a/router/src/infer/chat_template.rs
+++ b/router/src/infer/chat_template.rs
@@ -1,9 +1,8 @@
-use std::collections::HashSet;
-
 use crate::infer::InferError;
 use crate::{ChatTemplateInputs, Message, MessageChunk, TextMessage, TokenizerConfigToken, Tool};
 use minijinja::{Environment, ErrorKind, Template};
 use minijinja_contrib::pycompat;
+use std::collections::HashSet;
 
 /// Raise a exception (custom function) used in the chat templates
 pub(crate) fn raise_exception(err_text: String) -> Result<String, minijinja::Error> {
@@ -872,4 +871,38 @@ mod tests {
         let expected = "<s>[INST] I'd like to show off how chat templating works! [/INST]Great! How can I help you today?</s> [INST] Just testing\n---\n[{\"type\":\"function\",\"function\":{\"description\":\"Get the current weather\",\"name\":\"get_current_weather\",\"arguments\":{\"properties\":{\"format\":{\"description\":\"The temperature unit to use. Infer this from the users location.\",\"enum\":[\"celsius\",\"fahrenheit\"],\"type\":\"string\"},\"location\":{\"description\":\"The city and state, e.g. San Francisco, CA\",\"type\":\"string\"}},\"required\":[\"location\",\"format\"],\"type\":\"object\"}}}]\nThis default prompt will be used [/INST]".to_string();
         assert_eq!(result.unwrap(), expected);
     }
+
+    #[test]
+    fn test_chat_template_with_custom_tool_template() {
+        // chat template from meta-llama/Meta-Llama-3.1-8B-Instruct
+        let ct = ChatTemplate::new(
+            "{{- bos_token }}\n{%- if not tools_in_user_message is defined %}\n    {%- set tools_in_user_message = true %}\n{%- endif %}\n{%- if not date_string is defined %}\n    {%- set date_string = \"26 Jul 2024\" %}\n{%- endif %}\n{%- if not tools is defined %}\n    {%- set tools = none %}\n{%- endif %}\n\n{#- This block extracts the system message, so we can slot it into the right place. #}\n{%- if messages[0]['role'] == 'system' %}\n    {%- set system_message = messages[0]['content']|trim %}\n    {%- set messages = messages[1:] %}\n{%- else %}\n    {%- set system_message = \"\" %}\n{%- endif %}\n\n{#- System message + builtin tools #}\n{{- \"<|start_header_id|>system<|end_header_id|>\\n\\n\" }}\n{%- if builtin_tools is defined or tools is not none %}\n    {{- \"Environment: ipython\\n\" }}\n{%- endif %}\n{%- if builtin_tools is defined %}\n    {{- \"Tools: \" + builtin_tools | reject('equalto', 'code_interpreter') | join(\", \") + \"\\n\\n\"}}\n{%- endif %}\n{{- \"Cutting Knowledge Date: December 2023\\n\" }}\n{{- \"Today Date: \" + date_string + \"\\n\\n\" }}\n{%- if tools is not none and not tools_in_user_message %}\n    {{- \"You have access to the following functions. To call a function, please respond with JSON for a function call.\" }}\n    {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n    {{- \"Do not use variables.\\n\\n\" }}\n    {%- for t in tools %}\n        {{- t | tojson(indent=4) }}\n        {{- \"\\n\\n\" }}\n    {%- endfor %}\n{%- endif %}\n{{- system_message }}\n{{- \"<|eot_id|>\" }}\n\n{#- Custom tools are passed in a user message with some extra guidance #}\n{%- if tools_in_user_message and not tools is none %}\n    {#- Extract the first user message so we can plug it in here #}\n    {%- if messages | length != 0 %}\n        {%- set first_user_message = messages[0]['content']|trim %}\n        {%- set messages = messages[1:] %}\n    {%- else %}\n        {{- raise_exception(\"Cannot put tools in the first user message when there's no first user message!\") }}\n{%- endif %}\n    {{- '<|start_header_id|>user<|end_header_id|>\\n\\n' -}}\n    {{- \"Given the following functions, please respond with a JSON for a function call \" }}\n    {{- \"with its proper arguments that best answers the given prompt.\\n\\n\" }}\n    {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n    {{- \"Do not use variables.\\n\\n\" }}\n    {%- for t in tools %}\n        {{- t | tojson(indent=4) }}\n        {{- \"\\n\\n\" }}\n    {%- endfor %}\n    {{- first_user_message + \"<|eot_id|>\"}}\n{%- endif %}\n\n{%- for message in messages %}\n    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}\n        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\\n\\n'+ message['content'] | trim + '<|eot_id|>' }}\n    {%- elif 'tool_calls' in message %}\n        {%- if not message.tool_calls|length == 1 %}\n            {{- raise_exception(\"This model only supports single tool-calls at once!\") }}\n        {%- endif %}\n        {%- set tool_call = message.tool_calls[0].function %}\n        {%- if builtin_tools is defined and tool_call.name in builtin_tools %}\n            {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n            {{- \"<|python_tag|>\" + tool_call.name + \".call(\" }}\n            {%- for arg_name, arg_val in tool_call.arguments | items %}\n                {{- arg_name + '=\"' + arg_val + '\"' }}\n                {%- if not loop.last %}\n                    {{- \", \" }}\n                {%- endif %}\n                {%- endfor %}\n            {{- \")\" }}\n        {%- else  %}\n            {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n            {{- '{\"name\": \"' + tool_call.name + '\", ' }}\n            {{- '\"parameters\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- \"}\" }}\n        {%- endif %}\n        {%- if builtin_tools is defined %}\n            {#- This means we're in ipython mode #}\n            {{- \"<|eom_id|>\" }}\n        {%- else %}\n            {{- \"<|eot_id|>\" }}\n        {%- endif %}\n    {%- elif message.role == \"tool\" or message.role == \"ipython\" %}\n        {{- \"<|start_header_id|>ipython<|end_header_id|>\\n\\n\" }}\n        {%- if message.content is mapping or message.content is iterable %}\n            {{- message.content | tojson }}\n        {%- else %}\n            {{- message.content }}\n        {%- endif %}\n        {{- \"<|eot_id|>\" }}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' }}\n{%- endif %}\n".to_string(),
+            Some(TokenizerConfigToken::String("<s>".to_string())),
+            Some(TokenizerConfigToken::String("</s>".to_string())),
+        );
+        let msgs: Vec<Message> = vec![
+            Message {
+                name: None,
+                role: "system".to_string(),
+                content: MessageContent::SingleText(
+                    "Youre a helpful assistant! Answer the users question best you can."
+                        .to_string(),
+                ),
+            },
+            Message {
+                name: None,
+                role: "user".to_string(),
+                content: MessageContent::SingleText(
+                    "What is the weather like in Brooklyn, New York?".to_string(),
+                ),
+            },
+        ];
+        let tools_string = r#"[{"type": "function","function": {"name": "get_current_weather","description": "Get the current weather","parameters": {"type": "object","properties": {"location": {"type": "string","description": "The city and state, e.g. San Francisco, CA"},"format": {"type": "string","enum": ["celsius", "fahrenheit"],"description": "The temperature unit to use. Infer this from the users location."}},"required": ["location", "format"]}}}]"#.to_string();
+        let tools: Vec<Tool> = serde_json::from_str(&tools_string).unwrap();
+        let tool_prompt = "This default prompt will be used".to_string();
+        let tools_and_prompt = Some((tools, tool_prompt));
+        let result = ct.apply(None, msgs, tools_and_prompt);
+        let expected = "<s><|start_header_id|>system<|end_header_id|>\n\nEnvironment: ipython\nCutting Knowledge Date: December 2023\nToday Date: 26 Jul 2024\n\nYoure a helpful assistant! Answer the users question best you can.<|eot_id|><|start_header_id|>user<|end_header_id|>\n\nGiven the following functions, please respond with a JSON for a function call with its proper arguments that best answers the given prompt.\n\nRespond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.Do not use variables.\n\n{\n    \"function\": {\n        \"arguments\": {\n            \"properties\": {\n                \"format\": {\n                    \"description\": \"The temperature unit to use. Infer this from the users location.\",\n                    \"enum\": [\n                        \"celsius\",\n                        \"fahrenheit\"\n                    ],\n                    \"type\": \"string\"\n                },\n                \"location\": {\n                    \"description\": \"The city and state, e.g. San Francisco, CA\",\n                    \"type\": \"string\"\n                }\n            },\n            \"required\": [\n                \"location\",\n                \"format\"\n            ],\n            \"type\": \"object\"\n        },\n        \"description\": \"Get the current weather\",\n        \"name\": \"get_current_weather\"\n    },\n    \"type\": \"function\"\n}\n\nWhat is the weather like in Brooklyn, New York?\n---\nThis default prompt will be used<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n".to_string();
+        assert_eq!(result.unwrap(), expected);
+    }
 }

From 8f99f165ce1a261c89ea2edef437ef23c03a0716 Mon Sep 17 00:00:00 2001
From: drbh <david.richard.holtz@gmail.com>
Date: Wed, 28 Aug 2024 13:44:44 -0400
Subject: [PATCH 67/72] fix: improve regex expression (#2468)

---
 docs/source/basic_tutorials/using_guidance.md | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/docs/source/basic_tutorials/using_guidance.md b/docs/source/basic_tutorials/using_guidance.md
index f09ef348..dfa3f0e4 100644
--- a/docs/source/basic_tutorials/using_guidance.md
+++ b/docs/source/basic_tutorials/using_guidance.md
@@ -157,7 +157,12 @@ from huggingface_hub import InferenceClient
 
 client = InferenceClient("http://localhost:3000")
 
-regexp = "((25[0-5]|2[0-4]\\d|[01]?\\d\\d?)\\.){3}(25[0-5]|2[0-4]\\d|[01]?\\d\\d?)"
+section_regex = "(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)"
+regexp = f"HELLO\.{section_regex}\.WORLD\.{section_regex}"
+
+# This is a more realistic example of an ip address regex
+# regexp = f"{section_regex}\.{section_regex}\.{section_regex}\.{section_regex}"
+
 
 resp = client.text_generation(
     f"Whats Googles DNS? Please use the following regex: {regexp}",
@@ -170,7 +175,7 @@ resp = client.text_generation(
 
 
 print(resp)
-# 7.1.1.1
+# HELLO.255.WORLD.255
 
 ```
 

From 4e821c003a7cb055a358cf142dbf01a2f4c1916f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Dani=C3=ABl=20de=20Kok?= <me@danieldk.eu>
Date: Thu, 29 Aug 2024 16:25:25 +0200
Subject: [PATCH 68/72] nix: build Torch against MKL and various other
 improvements (#2469)

Updates tgi-nix input:

- Move Torch closer to upstream by building against MKL.
- Remove compute capability 8.7 from Torch (Jetson).
- Sync nixpkgs cumpute capabilities with Torch (avoids
  compiling too mana capabilities for MAGMA).
- Use nixpkgs configuration passed through by `tgi-nix`.
---
 flake.lock     | 6 +++---
 flake.nix      | 9 +++------
 nix/server.nix | 1 +
 3 files changed, 7 insertions(+), 9 deletions(-)

diff --git a/flake.lock b/flake.lock
index b40f51b3..14011768 100644
--- a/flake.lock
+++ b/flake.lock
@@ -944,11 +944,11 @@
         "nixpkgs": "nixpkgs_6"
       },
       "locked": {
-        "lastModified": 1724270760,
-        "narHash": "sha256-KX566x0+3HZcB20HPdvdwyMm7ZJg21M+iqVrs/HCimA=",
+        "lastModified": 1724784743,
+        "narHash": "sha256-NdEoWeNwR/ZstYnHaiQWIYZvr7VsrAh7g3+ZHUPrxuI=",
         "owner": "danieldk",
         "repo": "tgi-nix",
-        "rev": "12cbaa76ff258351741d3b5afb7161f617fe7b4c",
+        "rev": "c9580c3e39a855246bb87b584bbea1885b44f524",
         "type": "github"
       },
       "original": {
diff --git a/flake.nix b/flake.nix
index 83feb26a..0739c90a 100644
--- a/flake.nix
+++ b/flake.nix
@@ -31,15 +31,12 @@
           src = ./.;
           additionalCargoNixArgs = [ "--all-features" ];
         };
-        config = {
-          allowUnfree = true;
-          cudaSupport = true;
-        };
         pkgs = import nixpkgs {
-          inherit config system;
+          inherit system;
+          inherit (tgi-nix.lib) config;
           overlays = [
             rust-overlay.overlays.default
-            tgi-nix.overlay
+            tgi-nix.overlays.default
           ];
         };
         crateOverrides = import ./nix/crate-overrides.nix { inherit pkgs nix-filter; };
diff --git a/nix/server.nix b/nix/server.nix
index 4e0fdaa1..6ee088e0 100644
--- a/nix/server.nix
+++ b/nix/server.nix
@@ -28,6 +28,7 @@
   peft,
   safetensors,
   tokenizers,
+  torch,
   sentencepiece,
   transformers,
   typer,

From e415b690a68d7a0e149c996e46def41c867ff421 Mon Sep 17 00:00:00 2001
From: Nicolas Patry <patry.nicolas@protonmail.com>
Date: Thu, 29 Aug 2024 16:29:01 +0200
Subject: [PATCH 69/72] Lots of improvements (Still 2 allocators) (#2449)

* Making prefix/flashinfer the default and testing the full release tests.

* Include flashinfer in the docker.

* Using prebuilt.

* Allowing window_left_size (dummy version).

* Disabling flashinfer/prefix caching on odd head_dim

* Disable prefix caching for lora.

* More specific codes.

* Update lock

* Updating integration tests with new values with FI/FD.

Remove paged as a default too, and using FD everywhere.

* Update cargo lock ?

* Upgrade to 1.80 because of bitstream...

* Everywhere 1.80

* Forgot last default place.

* Apply suggestions from code review

Co-authored-by: drbh <david.richard.holtz@gmail.com>

* Updated flake lock

* Tmp

* Upgrade resolution system for less errors in resolution.

* Remove lambda for cleaner function.

* Handling debugger.

* OVerride the env in server tests.

* Is this enough to make it work ?

* This seems to be working.

* Downgrade some logs.

* Fixing the default for vlm.

* Don't enable prefix caching on VLM just yet.

* Change `add_special_tokens` in order to have the correct tokens for chat
input and not (since it's super important with the prefixing now)

* Fixing prefix caching for flashdecoding.

* Update all models.

* Fixed flashinfer version.

* add_special_tokens is internal only

* Fixing seqlen with the new vlms.

* Fixing the issue with `add_special_tokens` not being passed around.

* Fixing the test.

* Removing encoder_decoder (seq2seq).

* Update the chat test.

* Fixing the batching tokenization in flash causal lm.

* Truncating left for radix purposes.

* Oops this doesn't belong here.

* Put back default pure shell.

* Update server tests

- Default to throughput test in k6
- Use TGI_WIGGLE_ROOM to adjust wiggle room

* Only n_heads / process_group.size() are necessary.

* Revert the integrationt tests change (seem linked to head_size
modification).

* Adding error message when assert is violated.

* Fixing the free algorithm to handle times where the common prefix is
smaller.

* Apply suggestions from code review

Co-authored-by: OlivierDehaene <olivier@huggingface.co>

* Update server/text_generation_server/layers/attention/common.py

Co-authored-by: OlivierDehaene <olivier@huggingface.co>

* Fix disabling prefix caching - Fix windowing checks.

* Revert the Cohere tokenizer change (for now using a revision instead).

* Fmt.

---------

Co-authored-by: drbh <david.richard.holtz@gmail.com>
Co-authored-by: OlivierDehaene <olivier@huggingface.co>
---
 .github/workflows/tests.yaml                  |   2 +-
 Cargo.lock                                    | 437 +++++++++---------
 Dockerfile                                    |   9 +-
 Dockerfile_amd                                |   2 +-
 Dockerfile_intel                              |   2 +-
 backends/client/src/v3/client.rs              |   2 +
 backends/client/src/v3/sharded_client.rs      |   1 +
 backends/v3/src/backend.rs                    |  30 +-
 backends/v3/src/block_allocator.rs            |   5 +-
 backends/v3/src/client/grpc_client.rs         |   1 +
 backends/v3/src/client/sharded_client.rs      |   1 +
 backends/v3/src/queue.rs                      |   2 +
 backends/v3/src/radix.rs                      | 206 ++++++---
 benchmark/src/generation.rs                   |   1 +
 flake.lock                                    |   6 +-
 .../test_flash_llama_simple.json              |  12 +-
 ..._llama_completion_many_prompts_stream.json | 172 +++----
 .../test_flash_deepseek_v2.json               |  44 +-
 .../test_flash_deepseek_v2_load.json          | 152 +++---
 .../test_flash_llama_fp8_all_params.json      |  58 +--
 .../test_flash_starcoder2_default_params.json |  16 +-
 .../test_flash_idefics2_next_all_params.json  |   8 +-
 integration-tests/models/test_chat_llama.py   |   2 +-
 launcher/src/main.rs                          | 265 ++++++++---
 load_tests/common.js                          |  26 +-
 proto/v3/generate.proto                       |   2 +
 router/src/infer/mod.rs                       |   3 +-
 router/src/lib.rs                             |  21 +
 router/src/server.rs                          |   4 +
 router/src/validation.rs                      |  49 +-
 rust-toolchain.toml                           |   2 +-
 server/Makefile                               |   1 +
 server/Makefile-flashinfer                    |   2 +
 server/tests/conftest.py                      |   5 +-
 .../layers/attention/common.py                |  39 +-
 .../layers/attention/cuda.py                  |  28 +-
 .../text_generation_server/models/__init__.py |  15 +-
 .../custom_modeling/flash_cohere_modeling.py  |  23 +-
 .../custom_modeling/flash_dbrx_modeling.py    |  27 +-
 .../flash_deepseek_v2_modeling.py             |  24 +-
 .../custom_modeling/flash_gemma2_modeling.py  |  23 +-
 .../custom_modeling/flash_gemma_modeling.py   |  23 +-
 .../custom_modeling/flash_gpt2_modeling.py    |  23 +-
 .../custom_modeling/flash_gptj_modeling.py    |  25 +-
 .../custom_modeling/flash_llama_modeling.py   |  23 +-
 .../custom_modeling/flash_mistral_modeling.py |  25 +-
 .../custom_modeling/flash_mixtral_modeling.py |  25 +-
 .../custom_modeling/flash_neox_modeling.py    |  25 +-
 .../flash_pali_gemma_modeling.py              |   5 +-
 .../custom_modeling/flash_phi_modeling.py     |  23 +-
 .../custom_modeling/flash_qwen2_modeling.py   |  25 +-
 .../custom_modeling/flash_rw_modeling.py      |  33 +-
 .../flash_santacoder_modeling.py              |  23 +-
 .../flash_starcoder2_modeling.py              |  25 +-
 .../models/custom_modeling/idefics2.py        |   5 +-
 .../models/custom_modeling/llava_next.py      |   5 +-
 .../models/flash_causal_lm.py                 | 105 +++--
 .../text_generation_server/models/globals.py  |   9 +-
 .../models/vlm_causal_lm.py                   |  11 +-
 59 files changed, 1234 insertions(+), 934 deletions(-)
 create mode 100644 server/Makefile-flashinfer

diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
index f983b6ed..6faabe3b 100644
--- a/.github/workflows/tests.yaml
+++ b/.github/workflows/tests.yaml
@@ -35,7 +35,7 @@ jobs:
         with:
           # Released on: 02 May, 2024
           # https://releases.rs/docs/1.78.0/
-          toolchain: 1.79.0
+          toolchain: 1.80.0
           override: true
           components: rustfmt, clippy
       - name: Install Protoc
diff --git a/Cargo.lock b/Cargo.lock
index 02e91bc1..00c7f005 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -17,6 +17,12 @@ version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
 
+[[package]]
+name = "adler2"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627"
+
 [[package]]
 name = "ahash"
 version = "0.8.11"
@@ -28,7 +34,7 @@ dependencies = [
  "once_cell",
  "serde",
  "version_check",
- "zerocopy 0.7.35",
+ "zerocopy",
 ]
 
 [[package]]
@@ -121,14 +127,14 @@ checksum = "0ae92a5119aa49cdbcf6b9f893fe4e1d98b04ccbf82ee0584ad948a44a734dea"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.72",
+ "syn 2.0.76",
 ]
 
 [[package]]
 name = "arrayvec"
-version = "0.7.4"
+version = "0.7.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711"
+checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
 
 [[package]]
 name = "async-rustls"
@@ -160,7 +166,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.72",
+ "syn 2.0.76",
 ]
 
 [[package]]
@@ -171,7 +177,7 @@ checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.72",
+ "syn 2.0.76",
 ]
 
 [[package]]
@@ -257,9 +263,9 @@ dependencies = [
 
 [[package]]
 name = "aws-lc-sys"
-version = "0.20.0"
+version = "0.20.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2e89b6941c2d1a7045538884d6e760ccfffdf8e1ffc2613d8efa74305e1f3752"
+checksum = "0f0e249228c6ad2d240c2dc94b714d711629d52bad946075d8e9b2f5391f0703"
 dependencies = [
  "bindgen",
  "cc",
@@ -402,7 +408,7 @@ dependencies = [
  "cc",
  "cfg-if",
  "libc",
- "miniz_oxide",
+ "miniz_oxide 0.7.4",
  "object",
  "rustc-demangle",
 ]
@@ -444,7 +450,7 @@ dependencies = [
  "regex",
  "rustc-hash",
  "shlex",
- "syn 2.0.72",
+ "syn 2.0.76",
  "which",
 ]
 
@@ -483,9 +489,9 @@ checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de"
 
 [[package]]
 name = "bitstream-io"
-version = "2.5.0"
+version = "2.5.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3dcde5f311c85b8ca30c2e4198d4326bc342c76541590106f5fa4a50946ea499"
+checksum = "b81e1519b0d82120d2fd469d5bfb2919a9361c48b02d82d04befc1cdd2002452"
 
 [[package]]
 name = "block-buffer"
@@ -516,9 +522,9 @@ checksum = "5ce89b21cab1437276d2650d57e971f9d548a2d9037cc231abdc0562b97498ce"
 
 [[package]]
 name = "bytemuck"
-version = "1.16.1"
+version = "1.17.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b236fc92302c97ed75b38da1f4917b5cdda4984745740f153a5d3059e48d725e"
+checksum = "773d90827bc3feecfb67fab12e24de0749aad83c74b9504ecde46237b5cd24e2"
 
 [[package]]
 name = "byteorder"
@@ -534,15 +540,15 @@ checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495"
 
 [[package]]
 name = "bytes"
-version = "1.6.1"
+version = "1.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a12916984aab3fa6e39d655a33e09c0071eb36d6ab3aea5c2d78551f1df6d952"
+checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50"
 
 [[package]]
 name = "camino"
-version = "1.1.7"
+version = "1.1.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e0ec6b951b160caa93cc0c7b209e5a3bff7aae9062213451ac99493cd844c239"
+checksum = "8b96ec4966b5813e2c0507c1f86115c8c5abaadc3980879c3424042a02fd1ad3"
 dependencies = [
  "serde",
 ]
@@ -584,12 +590,13 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
 
 [[package]]
 name = "cc"
-version = "1.1.7"
+version = "1.1.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "26a5c3fd7bfa1ce3897a3a3501d362b2d87b7f2583ebcb4a949ec25911025cbc"
+checksum = "57b6a275aa2903740dc87da01c62040406b8812552e97129a63ea8850a17c6e6"
 dependencies = [
  "jobserver",
  "libc",
+ "shlex",
 ]
 
 [[package]]
@@ -623,6 +630,12 @@ version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e"
 
+[[package]]
+name = "cfg_aliases"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
+
 [[package]]
 name = "clang-sys"
 version = "1.8.1"
@@ -647,9 +660,9 @@ dependencies = [
 
 [[package]]
 name = "clap"
-version = "4.5.11"
+version = "4.5.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "35723e6a11662c2afb578bcf0b88bf6ea8e21282a953428f240574fcc3a2b5b3"
+checksum = "ed6719fffa43d0d87e5fd8caeab59be1554fb028cd30edc88fc4369b17971019"
 dependencies = [
  "clap_builder",
  "clap_derive",
@@ -657,9 +670,9 @@ dependencies = [
 
 [[package]]
 name = "clap_builder"
-version = "4.5.11"
+version = "4.5.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "49eb96cbfa7cfa35017b7cd548c75b14c3118c98b423041d70562665e07fb0fa"
+checksum = "216aec2b177652e3846684cbfe25c9964d18ec45234f0f5da5157b207ed1aab6"
 dependencies = [
  "anstream",
  "anstyle",
@@ -669,14 +682,14 @@ dependencies = [
 
 [[package]]
 name = "clap_derive"
-version = "4.5.11"
+version = "4.5.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5d029b67f89d30bbb547c89fd5161293c0aec155fc691d7924b64550662db93e"
+checksum = "501d359d5f3dcaf6ecdeee48833ae73ec6e42723a1e52419c79abf9507eec0a0"
 dependencies = [
  "heck 0.5.0",
  "proc-macro2",
  "quote",
- "syn 2.0.72",
+ "syn 2.0.76",
 ]
 
 [[package]]
@@ -687,9 +700,9 @@ checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97"
 
 [[package]]
 name = "cmake"
-version = "0.1.50"
+version = "0.1.51"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a31c789563b815f77f4250caee12365734369f942439b7defd71e18a48197130"
+checksum = "fb1e43aa7fd152b1f968787f7dbcdeb306d1867ff373c69955211876c053f91a"
 dependencies = [
  "cc",
 ]
@@ -741,15 +754,15 @@ dependencies = [
 
 [[package]]
 name = "core-foundation-sys"
-version = "0.8.6"
+version = "0.8.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f"
+checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
 
 [[package]]
 name = "cpufeatures"
-version = "0.2.12"
+version = "0.2.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504"
+checksum = "51e852e6dc9a5bed1fae92dd2375037bf2b768725bf3be87811edee3249d09ad"
 dependencies = [
  "libc",
 ]
@@ -897,19 +910,19 @@ dependencies = [
 
 [[package]]
 name = "ctrlc"
-version = "3.4.4"
+version = "3.4.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "672465ae37dc1bc6380a6547a8883d5dd397b0f1faaad4f265726cc7042a5345"
+checksum = "90eeab0aa92f3f9b4e87f258c72b139c207d251f9cbc1080a0086b86a8870dd3"
 dependencies = [
- "nix",
- "windows-sys 0.52.0",
+ "nix 0.29.0",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
 name = "cxx"
-version = "1.0.124"
+version = "1.0.126"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "273dcfd3acd4e1e276af13ed2a43eea7001318823e7a726a6b3ed39b4acc0b82"
+checksum = "3c4eae4b7fc8dcb0032eb3b1beee46b38d371cdeaf2d0c64b9944f6f69ad7755"
 dependencies = [
  "cc",
  "cxxbridge-flags",
@@ -919,9 +932,9 @@ dependencies = [
 
 [[package]]
 name = "cxx-build"
-version = "1.0.124"
+version = "1.0.126"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d8b2766fbd92be34e9ed143898fce6c572dc009de39506ed6903e5a05b68914e"
+checksum = "6c822bf7fb755d97328d6c337120b6f843678178751cba33c9da25cf522272e0"
 dependencies = [
  "cc",
  "codespan-reporting",
@@ -929,24 +942,24 @@ dependencies = [
  "proc-macro2",
  "quote",
  "scratch",
- "syn 2.0.72",
+ "syn 2.0.76",
 ]
 
 [[package]]
 name = "cxxbridge-flags"
-version = "1.0.124"
+version = "1.0.126"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "839fcd5e43464614ffaa989eaf1c139ef1f0c51672a1ed08023307fa1b909ccd"
+checksum = "719d6197dc016c88744aff3c0d0340a01ecce12e8939fc282e7c8f583ee64bc6"
 
 [[package]]
 name = "cxxbridge-macro"
-version = "1.0.124"
+version = "1.0.126"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4b2c1c1776b986979be68bb2285da855f8d8a35851a769fca8740df7c3d07877"
+checksum = "35de3b547387863c8f82013c4f79f1c2162edee956383e4089e1d04c18c4f16c"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.72",
+ "syn 2.0.76",
 ]
 
 [[package]]
@@ -970,7 +983,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "strsim",
- "syn 2.0.72",
+ "syn 2.0.76",
 ]
 
 [[package]]
@@ -981,7 +994,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806"
 dependencies = [
  "darling_core",
  "quote",
- "syn 2.0.72",
+ "syn 2.0.76",
 ]
 
 [[package]]
@@ -1011,7 +1024,7 @@ dependencies = [
  "darling",
  "proc-macro2",
  "quote",
- "syn 2.0.72",
+ "syn 2.0.76",
 ]
 
 [[package]]
@@ -1021,7 +1034,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "206868b8242f27cecce124c19fd88157fbd0dd334df2587f36417bafbc85097b"
 dependencies = [
  "derive_builder_core",
- "syn 2.0.72",
+ "syn 2.0.76",
 ]
 
 [[package]]
@@ -1057,9 +1070,9 @@ dependencies = [
 
 [[package]]
 name = "dunce"
-version = "1.0.4"
+version = "1.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "56ce8c6da7551ec6c462cbaf3bfbc75131ebbfa1c944aeaa9dab51ca1c5f0c3b"
+checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813"
 
 [[package]]
 name = "easy-cast"
@@ -1126,7 +1139,7 @@ dependencies = [
  "flume",
  "half 2.4.1",
  "lebe",
- "miniz_oxide",
+ "miniz_oxide 0.7.4",
  "rayon-core",
  "smallvec",
  "zune-inflate",
@@ -1144,9 +1157,9 @@ dependencies = [
 
 [[package]]
 name = "fastrand"
-version = "2.1.0"
+version = "2.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a"
+checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6"
 
 [[package]]
 name = "fdeflate"
@@ -1165,12 +1178,12 @@ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80"
 
 [[package]]
 name = "flate2"
-version = "1.0.30"
+version = "1.0.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5f54427cfd1c7829e2a139fcefea601bf088ebca651d2bf53ebc600eac295dae"
+checksum = "324a1be68054ef05ad64b861cc9eaf1d623d2d8cb25b4bf2cb9cdd902b4bf253"
 dependencies = [
  "crc32fast",
- "miniz_oxide",
+ "miniz_oxide 0.8.0",
 ]
 
 [[package]]
@@ -1296,7 +1309,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.72",
+ "syn 2.0.76",
 ]
 
 [[package]]
@@ -1405,7 +1418,7 @@ dependencies = [
  "futures-sink",
  "futures-util",
  "http 0.2.12",
- "indexmap 2.2.6",
+ "indexmap 2.4.0",
  "slab",
  "tokio",
  "tokio-util",
@@ -1414,9 +1427,9 @@ dependencies = [
 
 [[package]]
 name = "h2"
-version = "0.4.5"
+version = "0.4.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fa82e28a107a8cc405f0839610bdc9b15f1e25ec7d696aa5cf173edbcb1486ab"
+checksum = "524e8ac6999421f49a846c2d4411f337e53497d8ec55d67753beffa43c5d9205"
 dependencies = [
  "atomic-waker",
  "bytes",
@@ -1424,7 +1437,7 @@ dependencies = [
  "futures-core",
  "futures-sink",
  "http 1.1.0",
- "indexmap 2.2.6",
+ "indexmap 2.4.0",
  "slab",
  "tokio",
  "tokio-util",
@@ -1631,7 +1644,7 @@ dependencies = [
  "bytes",
  "futures-channel",
  "futures-util",
- "h2 0.4.5",
+ "h2 0.4.6",
  "http 1.1.0",
  "http-body 1.0.1",
  "httparse",
@@ -1689,9 +1702,9 @@ dependencies = [
 
 [[package]]
 name = "hyper-util"
-version = "0.1.6"
+version = "0.1.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3ab92f4f49ee4fb4f997c784b7a2e0fa70050211e0b6a287f898c3c9785ca956"
+checksum = "cde7055719c54e36e95e8719f95883f22072a48ede39db7fc17a4e1d5281e9b9"
 dependencies = [
  "bytes",
  "futures-channel",
@@ -1774,9 +1787,9 @@ dependencies = [
 
 [[package]]
 name = "indexmap"
-version = "2.2.6"
+version = "2.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26"
+checksum = "93ead53efc7ea8ed3cfb0c79fc8023fbb782a5432b52830b6518941cebe6505c"
 dependencies = [
  "equivalent",
  "hashbrown 0.14.5",
@@ -1832,7 +1845,7 @@ checksum = "c34819042dc3d3971c46c2190835914dfbe0c3c13f61449b2997f4e9722dfa60"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.72",
+ "syn 2.0.76",
 ]
 
 [[package]]
@@ -1915,9 +1928,9 @@ checksum = "f5d4a7da358eff58addd2877a45865158f0d78c911d43a5784ceb7bbf52833b0"
 
 [[package]]
 name = "js-sys"
-version = "0.3.69"
+version = "0.3.70"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d"
+checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a"
 dependencies = [
  "wasm-bindgen",
 ]
@@ -1932,7 +1945,7 @@ dependencies = [
  "anyhow",
  "base64 0.21.7",
  "bytecount",
- "clap 4.5.11",
+ "clap 4.5.16",
  "fancy-regex",
  "fraction",
  "getrandom",
@@ -1972,9 +1985,9 @@ checksum = "03087c2bad5e1034e8cace5926dec053fb3790248370865f5117a7d0213354c8"
 
 [[package]]
 name = "libc"
-version = "0.2.155"
+version = "0.2.158"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c"
+checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439"
 
 [[package]]
 name = "libfuzzer-sys"
@@ -2126,7 +2139,7 @@ dependencies = [
  "hyper 1.4.1",
  "hyper-rustls",
  "hyper-util",
- "indexmap 2.2.6",
+ "indexmap 2.4.0",
  "ipnet",
  "metrics",
  "metrics-util",
@@ -2179,9 +2192,9 @@ dependencies = [
 
 [[package]]
 name = "minijinja-contrib"
-version = "2.1.0"
+version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6853ef2340c668281c5ea86b04da2ebb2fc9e98a7185a887591de4cac945d5b5"
+checksum = "744a2b84dbd22398e347594ed2aef9d3f1b948934e3e6e94ef69ecd39d597f4b"
 dependencies = [
  "minijinja",
  "serde",
@@ -2203,6 +2216,15 @@ dependencies = [
  "simd-adler32",
 ]
 
+[[package]]
+name = "miniz_oxide"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1"
+dependencies = [
+ "adler2",
+]
+
 [[package]]
 name = "mio"
 version = "0.8.11"
@@ -2217,9 +2239,9 @@ dependencies = [
 
 [[package]]
 name = "mio"
-version = "1.0.1"
+version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4569e456d394deccd22ce1c1913e6ea0e54519f577285001215d33557431afe4"
+checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec"
 dependencies = [
  "hermit-abi 0.3.9",
  "libc",
@@ -2251,7 +2273,7 @@ checksum = "a7ce64b975ed4f123575d11afd9491f2e37bbd5813fbfbc0f09ae1fbddea74e0"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.72",
+ "syn 2.0.76",
 ]
 
 [[package]]
@@ -2341,7 +2363,19 @@ checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4"
 dependencies = [
  "bitflags 2.6.0",
  "cfg-if",
- "cfg_aliases",
+ "cfg_aliases 0.1.1",
+ "libc",
+]
+
+[[package]]
+name = "nix"
+version = "0.29.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46"
+dependencies = [
+ "bitflags 2.6.0",
+ "cfg-if",
+ "cfg_aliases 0.2.1",
  "libc",
 ]
 
@@ -2439,7 +2473,7 @@ checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.72",
+ "syn 2.0.76",
 ]
 
 [[package]]
@@ -2510,9 +2544,9 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
 
 [[package]]
 name = "object"
-version = "0.36.2"
+version = "0.36.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3f203fa8daa7bb185f760ae12bd8e097f63d17041dcdcaf675ac54cdf863170e"
+checksum = "27b64972346851a39438c60b341ebc01bba47464ae329e55cf343eb93964efd9"
 dependencies = [
  "memchr",
 ]
@@ -2574,7 +2608,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.72",
+ "syn 2.0.76",
 ]
 
 [[package]]
@@ -2613,7 +2647,7 @@ checksum = "1e32339a5dc40459130b3bd269e9892439f55b33e772d2a9d402a789baaf4e8a"
 dependencies = [
  "futures-core",
  "futures-sink",
- "indexmap 2.2.6",
+ "indexmap 2.4.0",
  "js-sys",
  "once_cell",
  "pin-project-lite",
@@ -2837,7 +2871,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db"
 dependencies = [
  "fixedbitset",
- "indexmap 2.2.6",
+ "indexmap 2.4.0",
 ]
 
 [[package]]
@@ -2857,7 +2891,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.72",
+ "syn 2.0.76",
 ]
 
 [[package]]
@@ -2916,7 +2950,7 @@ dependencies = [
  "crc32fast",
  "fdeflate",
  "flate2",
- "miniz_oxide",
+ "miniz_oxide 0.7.4",
 ]
 
 [[package]]
@@ -2933,21 +2967,21 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
 
 [[package]]
 name = "ppv-lite86"
-version = "0.2.18"
+version = "0.2.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dee4364d9f3b902ef14fab8a1ddffb783a1cb6b4bba3bfc1fa3922732c7de97f"
+checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04"
 dependencies = [
- "zerocopy 0.6.6",
+ "zerocopy",
 ]
 
 [[package]]
 name = "prettyplease"
-version = "0.2.20"
+version = "0.2.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e"
+checksum = "479cf940fbbb3426c32c5d5176f62ad57549a0bb84773423ba8be9d089f5faba"
 dependencies = [
  "proc-macro2",
- "syn 2.0.72",
+ "syn 2.0.76",
 ]
 
 [[package]]
@@ -2999,7 +3033,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8021cf59c8ec9c432cfc2526ac6b8aa508ecaf29cd415f271b8406c1b851c3fd"
 dependencies = [
  "quote",
- "syn 2.0.72",
+ "syn 2.0.76",
 ]
 
 [[package]]
@@ -3039,7 +3073,7 @@ dependencies = [
  "prost 0.12.6",
  "prost-types",
  "regex",
- "syn 2.0.72",
+ "syn 2.0.76",
  "tempfile",
 ]
 
@@ -3066,7 +3100,7 @@ dependencies = [
  "itertools 0.12.1",
  "proc-macro2",
  "quote",
- "syn 2.0.72",
+ "syn 2.0.76",
 ]
 
 [[package]]
@@ -3110,9 +3144,9 @@ checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3"
 
 [[package]]
 name = "quote"
-version = "1.0.36"
+version = "1.0.37"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7"
+checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af"
 dependencies = [
  "proc-macro2",
 ]
@@ -3201,9 +3235,9 @@ dependencies = [
 
 [[package]]
 name = "ravif"
-version = "0.11.9"
+version = "0.11.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5797d09f9bd33604689e87e8380df4951d4912f01b63f71205e2abd4ae25e6b6"
+checksum = "a8f0bfd976333248de2078d350bfdf182ff96e168a24d23d2436cef320dd4bdd"
 dependencies = [
  "avif-serialize",
  "imgref",
@@ -3264,9 +3298,9 @@ dependencies = [
 
 [[package]]
 name = "redox_users"
-version = "0.4.5"
+version = "0.4.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bd283d9651eeda4b2a83a43c1c91b266c40fd76ecd39a50a8c630ae69dc72891"
+checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43"
 dependencies = [
  "getrandom",
  "libredox",
@@ -3275,9 +3309,9 @@ dependencies = [
 
 [[package]]
 name = "regex"
-version = "1.10.5"
+version = "1.10.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f"
+checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -3359,9 +3393,9 @@ dependencies = [
 
 [[package]]
 name = "rgb"
-version = "0.8.45"
+version = "0.8.48"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ade4539f42266ded9e755c605bdddf546242b2c961b03b06a7375260788a0523"
+checksum = "0f86ae463694029097b846d8f99fd5536740602ae00022c0c50c5600720b2f71"
 dependencies = [
  "bytemuck",
 ]
@@ -3416,7 +3450,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "rust-embed-utils",
- "syn 2.0.72",
+ "syn 2.0.76",
  "walkdir",
 ]
 
@@ -3507,12 +3541,12 @@ dependencies = [
 
 [[package]]
 name = "rustls-native-certs"
-version = "0.7.1"
+version = "0.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a88d6d420651b496bdd98684116959239430022a115c1240e6c3993be0b15fba"
+checksum = "04182dffc9091a404e0fc069ea5cd60e5b866c3adf881eff99a32d048242dffa"
 dependencies = [
  "openssl-probe",
- "rustls-pemfile 2.1.2",
+ "rustls-pemfile 2.1.3",
  "rustls-pki-types",
  "schannel",
  "security-framework",
@@ -3529,9 +3563,9 @@ dependencies = [
 
 [[package]]
 name = "rustls-pemfile"
-version = "2.1.2"
+version = "2.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "29993a25686778eb88d4189742cd713c9bce943bc54251a33509dc63cbacf73d"
+checksum = "196fe16b00e106300d3e45ecfcb764fa292a535d7326a29a5875c579c7417425"
 dependencies = [
  "base64 0.22.1",
  "rustls-pki-types",
@@ -3539,15 +3573,15 @@ dependencies = [
 
 [[package]]
 name = "rustls-pki-types"
-version = "1.7.0"
+version = "1.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d"
+checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0"
 
 [[package]]
 name = "rustls-webpki"
-version = "0.102.6"
+version = "0.102.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e6b52d4fda176fd835fdc55a835d4a89b8499cad995885a21149d5ad62f852e"
+checksum = "84678086bd54edf2b415183ed7a94d0efb049f1b646a33e22a36f3794be6ae56"
 dependencies = [
  "aws-lc-rs",
  "ring 0.17.8",
@@ -3641,9 +3675,9 @@ dependencies = [
 
 [[package]]
 name = "serde"
-version = "1.0.204"
+version = "1.0.209"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12"
+checksum = "99fce0ffe7310761ca6bf9faf5115afbc19688edd00171d81b1bb1b116c63e09"
 dependencies = [
  "serde_derive",
 ]
@@ -3660,20 +3694,20 @@ dependencies = [
 
 [[package]]
 name = "serde_derive"
-version = "1.0.204"
+version = "1.0.209"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222"
+checksum = "a5831b979fd7b5439637af1752d535ff49f4860c0f341d1baeb6faf0f4242170"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.72",
+ "syn 2.0.76",
 ]
 
 [[package]]
 name = "serde_json"
-version = "1.0.121"
+version = "1.0.127"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4ab380d7d9f22ef3f21ad3e6c1ebe8e4fc7a2000ccba2e4d71fc96f15b2cb609"
+checksum = "8043c06d9f82bd7271361ed64f415fe5e12a77fdb52e573e7f06a516dea329ad"
 dependencies = [
  "itoa",
  "memchr",
@@ -3875,7 +3909,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "rustversion",
- "syn 2.0.72",
+ "syn 2.0.76",
 ]
 
 [[package]]
@@ -3897,9 +3931,9 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "2.0.72"
+version = "2.0.76"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc4b9b9bf2add8093d3f2c0204471e951b2285580335de42f9d2534f3ae7a8af"
+checksum = "578e081a14e0cefc3279b0472138c513f37b41a08d5a3cca9b6e4e8ceb6cd525"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -3993,20 +4027,21 @@ dependencies = [
 
 [[package]]
 name = "target-lexicon"
-version = "0.12.15"
+version = "0.12.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4873307b7c257eddcb50c9bedf158eb669578359fb28428bef438fec8e6ba7c2"
+checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1"
 
 [[package]]
 name = "tempfile"
-version = "3.10.1"
+version = "3.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1"
+checksum = "04cbcdd0c794ebb0d4cf35e88edd2f7d2c4c3e9a5a6dab322839b321c6a87a64"
 dependencies = [
  "cfg-if",
  "fastrand",
+ "once_cell",
  "rustix",
- "windows-sys 0.52.0",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -4024,7 +4059,7 @@ version = "2.2.1-dev0"
 dependencies = [
  "async-stream",
  "async-trait",
- "clap 4.5.11",
+ "clap 4.5.16",
  "cmake",
  "cxx",
  "cxx-build",
@@ -4046,7 +4081,7 @@ name = "text-generation-benchmark"
 version = "2.2.1-dev0"
 dependencies = [
  "average",
- "clap 4.5.11",
+ "clap 4.5.16",
  "crossterm",
  "float-ord",
  "hf-hub",
@@ -4084,11 +4119,11 @@ dependencies = [
 name = "text-generation-launcher"
 version = "2.2.1-dev0"
 dependencies = [
- "clap 4.5.11",
+ "clap 4.5.16",
  "ctrlc",
  "float_eq",
  "hf-hub",
- "nix",
+ "nix 0.28.0",
  "once_cell",
  "reqwest",
  "serde",
@@ -4108,7 +4143,7 @@ dependencies = [
  "axum 0.7.5",
  "axum-tracing-opentelemetry",
  "base64 0.22.1",
- "clap 4.5.11",
+ "clap 4.5.16",
  "csv",
  "futures",
  "futures-util",
@@ -4156,7 +4191,7 @@ dependencies = [
  "axum 0.7.5",
  "axum-tracing-opentelemetry",
  "base64 0.22.1",
- "clap 4.5.11",
+ "clap 4.5.16",
  "criterion",
  "futures",
  "futures-util",
@@ -4224,7 +4259,7 @@ checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.72",
+ "syn 2.0.76",
 ]
 
 [[package]]
@@ -4341,14 +4376,14 @@ dependencies = [
 
 [[package]]
 name = "tokio"
-version = "1.39.2"
+version = "1.39.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "daa4fb1bc778bd6f04cbfc4bb2d06a7396a8f299dc33ea1900cedaa316f467b1"
+checksum = "9babc99b9923bfa4804bd74722ff02c0381021eafa4db9949217e3be8e84fff5"
 dependencies = [
  "backtrace",
  "bytes",
  "libc",
- "mio 1.0.1",
+ "mio 1.0.2",
  "parking_lot",
  "pin-project-lite",
  "signal-hook-registry",
@@ -4375,7 +4410,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.72",
+ "syn 2.0.76",
 ]
 
 [[package]]
@@ -4437,9 +4472,9 @@ dependencies = [
 
 [[package]]
 name = "toml"
-version = "0.8.16"
+version = "0.8.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "81967dd0dd2c1ab0bc3468bd7caecc32b8a4aa47d0c8c695d8c2b2108168d62c"
+checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e"
 dependencies = [
  "serde",
  "serde_spanned",
@@ -4449,20 +4484,20 @@ dependencies = [
 
 [[package]]
 name = "toml_datetime"
-version = "0.6.7"
+version = "0.6.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f8fb9f64314842840f1d940ac544da178732128f1c78c21772e876579e0da1db"
+checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41"
 dependencies = [
  "serde",
 ]
 
 [[package]]
 name = "toml_edit"
-version = "0.22.17"
+version = "0.22.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8d9f8729f5aea9562aac1cc0441f5d6de3cff1ee0c5d67293eeca5eb36ee7c16"
+checksum = "583c44c02ad26b0c3f3066fe629275e50627026c51ac2e595cca4c230ce1ce1d"
 dependencies = [
- "indexmap 2.2.6",
+ "indexmap 2.4.0",
  "serde",
  "serde_spanned",
  "toml_datetime",
@@ -4534,7 +4569,7 @@ dependencies = [
  "proc-macro2",
  "prost-build",
  "quote",
- "syn 2.0.72",
+ "syn 2.0.76",
 ]
 
 [[package]]
@@ -4575,15 +4610,15 @@ dependencies = [
 
 [[package]]
 name = "tower-layer"
-version = "0.3.2"
+version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0"
+checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e"
 
 [[package]]
 name = "tower-service"
-version = "0.3.2"
+version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52"
+checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3"
 
 [[package]]
 name = "tracing"
@@ -4605,7 +4640,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.72",
+ "syn 2.0.76",
 ]
 
 [[package]]
@@ -4865,7 +4900,7 @@ version = "4.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c5afb1a60e207dca502682537fefcfd9921e71d0b83e9576060f09abc6efab23"
 dependencies = [
- "indexmap 2.2.6",
+ "indexmap 2.4.0",
  "serde",
  "serde_json",
  "utoipa-gen",
@@ -4881,7 +4916,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "regex",
- "syn 2.0.72",
+ "syn 2.0.76",
 ]
 
 [[package]]
@@ -4919,7 +4954,7 @@ checksum = "ee1cd046f83ea2c4e920d6ee9f7c3537ef928d75dce5d84a87c2c5d6b3999a3a"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.72",
+ "syn 2.0.76",
 ]
 
 [[package]]
@@ -5000,34 +5035,35 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
 
 [[package]]
 name = "wasm-bindgen"
-version = "0.2.92"
+version = "0.2.93"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8"
+checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5"
 dependencies = [
  "cfg-if",
+ "once_cell",
  "wasm-bindgen-macro",
 ]
 
 [[package]]
 name = "wasm-bindgen-backend"
-version = "0.2.92"
+version = "0.2.93"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da"
+checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b"
 dependencies = [
  "bumpalo",
  "log",
  "once_cell",
  "proc-macro2",
  "quote",
- "syn 2.0.72",
+ "syn 2.0.76",
  "wasm-bindgen-shared",
 ]
 
 [[package]]
 name = "wasm-bindgen-futures"
-version = "0.4.42"
+version = "0.4.43"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0"
+checksum = "61e9300f63a621e96ed275155c108eb6f843b6a26d053f122ab69724559dc8ed"
 dependencies = [
  "cfg-if",
  "js-sys",
@@ -5037,9 +5073,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro"
-version = "0.2.92"
+version = "0.2.93"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726"
+checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf"
 dependencies = [
  "quote",
  "wasm-bindgen-macro-support",
@@ -5047,28 +5083,28 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro-support"
-version = "0.2.92"
+version = "0.2.93"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7"
+checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.72",
+ "syn 2.0.76",
  "wasm-bindgen-backend",
  "wasm-bindgen-shared",
 ]
 
 [[package]]
 name = "wasm-bindgen-shared"
-version = "0.2.92"
+version = "0.2.93"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96"
+checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484"
 
 [[package]]
 name = "web-sys"
-version = "0.3.69"
+version = "0.3.70"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef"
+checksum = "26fdeaafd9bd129f65e7c031593c24d62186301e0c72c8978fa1678be7d532c0"
 dependencies = [
  "js-sys",
  "wasm-bindgen",
@@ -5149,11 +5185,11 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
 
 [[package]]
 name = "winapi-util"
-version = "0.1.8"
+version = "0.1.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b"
+checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb"
 dependencies = [
- "windows-sys 0.52.0",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -5208,6 +5244,15 @@ dependencies = [
  "windows-targets 0.52.6",
 ]
 
+[[package]]
+name = "windows-sys"
+version = "0.59.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
+dependencies = [
+ "windows-targets 0.52.6",
+]
+
 [[package]]
 name = "windows-targets"
 version = "0.42.2"
@@ -5388,9 +5433,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
 
 [[package]]
 name = "winnow"
-version = "0.6.16"
+version = "0.6.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b480ae9340fc261e6be3e95a1ba86d54ae3f9171132a73ce8d4bbaf68339507c"
+checksum = "68a9bda4691f099d435ad181000724da8e5899daa10713c2d432552b9ccd3a6f"
 dependencies = [
  "memchr",
 ]
@@ -5405,34 +5450,14 @@ dependencies = [
  "windows-sys 0.48.0",
 ]
 
-[[package]]
-name = "zerocopy"
-version = "0.6.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "854e949ac82d619ee9a14c66a1b674ac730422372ccb759ce0c39cabcf2bf8e6"
-dependencies = [
- "byteorder",
- "zerocopy-derive 0.6.6",
-]
-
 [[package]]
 name = "zerocopy"
 version = "0.7.35"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0"
 dependencies = [
- "zerocopy-derive 0.7.35",
-]
-
-[[package]]
-name = "zerocopy-derive"
-version = "0.6.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "125139de3f6b9d625c39e2efdd73d41bdac468ccd556556440e322be0e1bbd91"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.72",
+ "byteorder",
+ "zerocopy-derive",
 ]
 
 [[package]]
@@ -5443,7 +5468,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.72",
+ "syn 2.0.76",
 ]
 
 [[package]]
@@ -5463,7 +5488,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.72",
+ "syn 2.0.76",
 ]
 
 [[package]]
diff --git a/Dockerfile b/Dockerfile
index 4c64a643..0d0e89b1 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,5 +1,5 @@
 # Rust builder
-FROM lukemathwalker/cargo-chef:latest-rust-1.79 AS chef
+FROM lukemathwalker/cargo-chef:latest-rust-1.80 AS chef
 WORKDIR /usr/src
 
 ARG CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse
@@ -184,6 +184,12 @@ WORKDIR /usr/src
 COPY server/Makefile-selective-scan Makefile
 RUN make build-all
 
+# Build flashinfer
+FROM kernel-builder AS flashinfer-builder
+WORKDIR /usr/src
+COPY server/Makefile-flashinfer Makefile
+RUN make install-flashinfer
+
 # Text Generation Inference base image
 FROM nvidia/cuda:12.1.0-base-ubuntu22.04 AS base
 
@@ -236,6 +242,7 @@ COPY --from=vllm-builder /usr/src/vllm/build/lib.linux-x86_64-cpython-310 /opt/c
 # Copy build artifacts from mamba builder
 COPY --from=mamba-builder /usr/src/mamba/build/lib.linux-x86_64-cpython-310/ /opt/conda/lib/python3.10/site-packages
 COPY --from=mamba-builder /usr/src/causal-conv1d/build/lib.linux-x86_64-cpython-310/ /opt/conda/lib/python3.10/site-packages
+COPY --from=flashinfer-builder /opt/conda/lib/python3.10/site-packages/flashinfer/ /opt/conda/lib/python3.10/site-packages/flashinfer/
 
 # Install flash-attention dependencies
 RUN pip install einops --no-cache-dir
diff --git a/Dockerfile_amd b/Dockerfile_amd
index cdad0d28..8cb699dd 100644
--- a/Dockerfile_amd
+++ b/Dockerfile_amd
@@ -1,5 +1,5 @@
 # Rust builder
-FROM lukemathwalker/cargo-chef:latest-rust-1.79 AS chef
+FROM lukemathwalker/cargo-chef:latest-rust-1.80 AS chef
 WORKDIR /usr/src
 
 ARG CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse
diff --git a/Dockerfile_intel b/Dockerfile_intel
index 12480c70..9af6422c 100644
--- a/Dockerfile_intel
+++ b/Dockerfile_intel
@@ -1,6 +1,6 @@
 ARG PLATFORM=xpu
 
-FROM lukemathwalker/cargo-chef:latest-rust-1.79 AS chef
+FROM lukemathwalker/cargo-chef:latest-rust-1.80 AS chef
 WORKDIR /usr/src
 
 ARG CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse
diff --git a/backends/client/src/v3/client.rs b/backends/client/src/v3/client.rs
index b321278c..479d31bf 100644
--- a/backends/client/src/v3/client.rs
+++ b/backends/client/src/v3/client.rs
@@ -153,6 +153,8 @@ impl Client {
                 }),
                 // We truncate the input on the server side to be sure that it has the correct size
                 truncate,
+                // Most request will have that
+                add_special_tokens: true,
                 // Blocks and slots will be set on the server side if we use paged attention
                 blocks: vec![],
                 slots: vec![],
diff --git a/backends/client/src/v3/sharded_client.rs b/backends/client/src/v3/sharded_client.rs
index 1cc173e3..645c076a 100644
--- a/backends/client/src/v3/sharded_client.rs
+++ b/backends/client/src/v3/sharded_client.rs
@@ -221,6 +221,7 @@ impl Health for ShardedClient {
                 chunks: vec![Chunk::Text("liveness".into()).into()],
             }),
             truncate: 10,
+            add_special_tokens: true,
             prefill_logprobs: false,
             parameters: Some(NextTokenChooserParameters {
                 temperature: 1.0,
diff --git a/backends/v3/src/backend.rs b/backends/v3/src/backend.rs
index cbcbff72..05a26370 100644
--- a/backends/v3/src/backend.rs
+++ b/backends/v3/src/backend.rs
@@ -35,27 +35,15 @@ impl BackendV3 {
         window_size: Option<u32>,
         speculate: u32,
     ) -> Self {
-        let prefix_caching = if let Ok(prefix_caching) = std::env::var("USE_PREFIX_CACHING") {
-            matches!(prefix_caching.as_str(), "true" | "1")
-        } else {
-            false
-        };
-        let attention = if let Ok(attention) = std::env::var("ATTENTION") {
-            attention
-                .parse()
-                .unwrap_or_else(|_| panic!("Invalid attention was specified :`{attention}`"))
-        } else if prefix_caching {
-            Attention::FlashInfer
-        } else {
-            Attention::Paged
-        };
-        let block_size = if attention == Attention::FlashDecoding {
-            256
-        } else if attention == Attention::FlashInfer {
-            1
-        } else {
-            16
-        };
+        let prefix_caching =
+            std::env::var("USE_PREFIX_CACHING").expect("Expect prefix caching env var");
+        let prefix_caching = matches!(prefix_caching.as_str(), "true" | "1");
+        let attention: String = std::env::var("ATTENTION").expect("attention env var");
+
+        let attention: Attention = attention
+            .parse()
+            .unwrap_or_else(|_| panic!("Invalid attention was specified :`{attention}`"));
+        let block_size = attention.block_size();
 
         let queue = Queue::new(
             requires_padding,
diff --git a/backends/v3/src/block_allocator.rs b/backends/v3/src/block_allocator.rs
index c5503b8c..4fea172b 100644
--- a/backends/v3/src/block_allocator.rs
+++ b/backends/v3/src/block_allocator.rs
@@ -1,4 +1,4 @@
-use std::{cmp::min, sync::Arc};
+use std::sync::Arc;
 use tokio::sync::{mpsc, oneshot};
 
 use crate::radix::RadixAllocator;
@@ -137,7 +137,6 @@ pub trait Allocator {
 
     fn free(&mut self, blocks: Vec<u32>, allocation_id: u64);
 }
-
 pub struct SimpleAllocator {
     free_blocks: Vec<u32>,
     block_size: u32,
@@ -167,7 +166,7 @@ impl Allocator for SimpleAllocator {
                 None => (tokens, 1),
                 Some(window_size) => {
                     let repeats = (tokens + window_size - 1) / window_size;
-                    let tokens = min(tokens, window_size);
+                    let tokens = core::cmp::min(tokens, window_size);
                     (tokens, repeats as usize)
                 }
             };
diff --git a/backends/v3/src/client/grpc_client.rs b/backends/v3/src/client/grpc_client.rs
index 6282759e..648662db 100644
--- a/backends/v3/src/client/grpc_client.rs
+++ b/backends/v3/src/client/grpc_client.rs
@@ -149,6 +149,7 @@ impl Client {
             requests.push(Request {
                 id: 0,
                 inputs,
+                add_special_tokens: true,
                 input_chunks: Some(Input {
                     chunks: input_chunks,
                 }),
diff --git a/backends/v3/src/client/sharded_client.rs b/backends/v3/src/client/sharded_client.rs
index 2f78da03..ea77a696 100644
--- a/backends/v3/src/client/sharded_client.rs
+++ b/backends/v3/src/client/sharded_client.rs
@@ -222,6 +222,7 @@ impl Health for ShardedClient {
                 chunks: vec![Chunk::Text("liveness".into()).into()],
             }),
             truncate: 10,
+            add_special_tokens: true,
             prefill_logprobs: false,
             parameters: Some(NextTokenChooserParameters {
                 temperature: 1.0,
diff --git a/backends/v3/src/queue.rs b/backends/v3/src/queue.rs
index faa57c11..2a8c4c53 100644
--- a/backends/v3/src/queue.rs
+++ b/backends/v3/src/queue.rs
@@ -383,6 +383,7 @@ impl State {
                 }),
                 inputs: entry.request.inputs.chunks_to_string(),
                 truncate: entry.request.truncate,
+                add_special_tokens: entry.request.add_special_tokens,
                 parameters: Some(NextTokenChooserParameters::from(
                     entry.request.parameters.clone(),
                 )),
@@ -517,6 +518,7 @@ mod tests {
                 inputs: vec![],
                 input_ids: Some(Arc::new(vec![])),
                 input_length: 0,
+                add_special_tokens: true,
                 truncate: 0,
                 decoder_input_details: false,
                 parameters: ValidParameters {
diff --git a/backends/v3/src/radix.rs b/backends/v3/src/radix.rs
index 5bac1a31..b85be00b 100644
--- a/backends/v3/src/radix.rs
+++ b/backends/v3/src/radix.rs
@@ -1,12 +1,10 @@
+use crate::block_allocator::{Allocator, BlockAllocation};
+use slotmap::{DefaultKey, SlotMap};
 use std::{
     collections::{BTreeSet, HashMap},
     sync::Arc,
 };
 
-use slotmap::{DefaultKey, SlotMap};
-
-use crate::block_allocator::{Allocator, BlockAllocation};
-
 pub struct RadixAllocator {
     allocation_id: u64,
 
@@ -16,26 +14,26 @@ pub struct RadixAllocator {
 
     /// Blocks that are immediately available for allocation.
     free_blocks: Vec<u32>,
+
+    #[allow(dead_code)]
+    // This isn't used because the prefix need to match without the windowing
+    // mecanism. This at worst is overallocating, not necessarily being wrong.
+    window_size: Option<u32>,
+
+    block_size: u32,
 }
 
 impl RadixAllocator {
     pub fn new(block_size: u32, n_blocks: u32, window_size: Option<u32>) -> Self {
-        assert_eq!(
-            block_size, 1,
-            "Radix tree allocator only works with block_size=1, was: {}",
-            block_size
-        );
-        if window_size.is_some() {
-            unimplemented!("Window size not supported in the prefix-caching block allocator yet");
-        }
-
         RadixAllocator {
             allocation_id: 0,
             allocations: HashMap::new(),
-            cache_blocks: RadixTrie::new(),
+            cache_blocks: RadixTrie::new(block_size as usize),
 
             // Block 0 is reserved for health checks.
             free_blocks: (1..n_blocks).collect(),
+            window_size,
+            block_size,
         }
     }
 
@@ -63,6 +61,7 @@ impl RadixAllocator {
     }
 }
 
+// Allocator trait
 impl Allocator for RadixAllocator {
     fn allocate(
         &mut self,
@@ -86,10 +85,12 @@ impl Allocator for RadixAllocator {
             .incref(prefix_node)
             .expect("Failed to increment refcount");
 
-        let prefix_len = blocks.len();
+        let prefix_len = blocks.len() * self.block_size as usize;
         let suffix_len = tokens - prefix_len as u32;
 
-        match self.alloc_or_reclaim(suffix_len as usize) {
+        let suffix_blocks = (suffix_len + self.block_size - 1) / self.block_size;
+
+        match self.alloc_or_reclaim(suffix_blocks as usize) {
             Some(suffix_blocks) => blocks.extend(suffix_blocks),
             None => {
                 self.cache_blocks
@@ -100,7 +101,20 @@ impl Allocator for RadixAllocator {
         }
 
         // 1:1 mapping of blocks and slots.
-        let slots = blocks.clone();
+        let slots = if self.block_size == 1 {
+            blocks.clone()
+        } else {
+            let mut slots = Vec::with_capacity(blocks.len() * self.block_size as usize);
+            'slots: for block_id in &blocks {
+                for s in (block_id * self.block_size)..((block_id + 1) * self.block_size) {
+                    slots.push(s);
+                    if slots.len() as u32 == tokens {
+                        break 'slots;
+                    }
+                }
+            }
+            slots
+        };
 
         let allocation = RadixAllocation {
             prefix_node,
@@ -108,6 +122,8 @@ impl Allocator for RadixAllocator {
             prefill_tokens: prefill_tokens.clone(),
         };
 
+        tracing::debug!("Blocks {blocks:?}");
+
         self.allocation_id += 1;
         self.allocations.insert(self.allocation_id, allocation);
 
@@ -136,27 +152,38 @@ impl Allocator for RadixAllocator {
             // If there are prefill tokens that did not come from the cache,
             // add them to the cache.
             if prefill_tokens.len() > allocation.cached_prefix_len {
-                let prefix_len = self
-                    .cache_blocks
-                    .insert(prefill_tokens, &blocks[..prefill_tokens.len()])
-                    // Unwrap, failing is a programming error.
-                    .expect("Failed to store prefill tokens");
-
-                // We can have a prefill with the following structure:
-                //
-                // |---| From the prefix cache.
-                // A B C D E F G
-                //|--------| Found in the trie during insertion.
-                //
-                // This means that while processing this request there was a
-                // partially overlapping request that had A..=E in its
-                // prefill. In this case we need to free the blocks D E.
-                self.free_blocks
-                    .extend(&blocks[allocation.cached_prefix_len..prefix_len]);
+                let aligned =
+                    (prefill_tokens.len() / self.block_size as usize) * self.block_size as usize;
+                if aligned > 0 {
+                    let prefix_len = self
+                        .cache_blocks
+                        .insert(
+                            &prefill_tokens[..aligned],
+                            &blocks[..aligned / self.block_size as usize],
+                        )
+                        // Unwrap, failing is a programming error.
+                        .expect("Failed to store prefill tokens");
+                    // We can have a prefill with the following structure:
+                    //
+                    // |---| From the prefix cache.
+                    // A B C D E F G
+                    //|--------| Found in the trie during insertion.
+                    //
+                    // This means that while processing this request there was a
+                    // partially overlapping request that had A..=E in its
+                    // prefill. In this case we need to free the blocks D E.
+                    if prefix_len > allocation.cached_prefix_len {
+                        self.free_blocks.extend(
+                            &blocks[allocation.cached_prefix_len / self.block_size as usize
+                                ..prefix_len / self.block_size as usize],
+                        );
+                    }
+                }
             }
 
             // Free non-prefill blocks.
-            self.free_blocks.extend(&blocks[prefill_tokens.len()..]);
+            self.free_blocks
+                .extend(&blocks[prefill_tokens.len() / self.block_size as usize..]);
         } else {
             self.free_blocks.extend(blocks);
         }
@@ -204,17 +231,14 @@ pub struct RadixTrie {
     /// Time as a monotonically increating counter to avoid the system
     /// call that a real time lookup would require.
     time: u64,
-}
 
-impl Default for RadixTrie {
-    fn default() -> Self {
-        Self::new()
-    }
+    /// All blocks need to be aligned with this
+    block_size: usize,
 }
 
 impl RadixTrie {
     /// Construct a new radix trie.
-    pub fn new() -> Self {
+    pub fn new(block_size: usize) -> Self {
         let root = TrieNode::new(vec![], vec![], 0, None);
         let mut nodes = SlotMap::new();
         let root = nodes.insert(root);
@@ -223,13 +247,14 @@ impl RadixTrie {
             nodes,
             root,
             time: 0,
+            block_size,
         }
     }
 
     /// Find the prefix of the given tokens.
     ///
     /// The blocks corresponding to the part of the prefix that could be found
-    /// are writteng to `blocks`. The number of blocks is in `0..=tokens.len()`.
+    /// are written to `blocks`. The number of blocks is in `0..=tokens.len()`.
     /// Returns the identifier of the trie node that contains the longest
     /// prefix. The node identifier can be used by callers to e.g. increase its
     /// reference count.
@@ -247,8 +272,9 @@ impl RadixTrie {
         if let Some(&child_id) = node.children.get(&key[0]) {
             self.update_access_time(child_id);
             let child = self.nodes.get(child_id).expect("Invalid child identifier");
-            let shared_prefix_len = child.key.shared_prefix_len(key);
-            blocks.extend(&child.blocks[..shared_prefix_len]);
+            let shared_prefix_len = shared_prefix(&child.key, key, self.block_size);
+            assert_eq!(shared_prefix_len % self.block_size, 0);
+            blocks.extend(&child.blocks[..shared_prefix_len / self.block_size]);
 
             let key = &key[shared_prefix_len..];
             if !key.is_empty() {
@@ -349,7 +375,8 @@ impl RadixTrie {
     /// the first 10 elements of the tree **the blocks are not updated**.
     pub fn insert(&mut self, tokens: &[u32], blocks: &[u32]) -> Result<usize, TrieError> {
         self.time += 1;
-        self.insert_(self.root, tokens, blocks)
+        let common = self.insert_(self.root, tokens, blocks)?;
+        Ok(common)
     }
 
     /// Insertion worker.
@@ -363,7 +390,7 @@ impl RadixTrie {
         // the part of the prefix that is already in the trie to detect
         // mismatches.
 
-        if tokens.len() != blocks.len() {
+        if tokens.len() != blocks.len() * self.block_size {
             return Err(TrieError::BlockTokenCountMismatch);
         }
 
@@ -374,10 +401,10 @@ impl RadixTrie {
                 .get_mut(child_id)
                 // Unwrap here, since failure is a bug.
                 .expect("Child node does not exist");
-            let shared_prefix_len = child.key.shared_prefix_len(tokens);
+            let shared_prefix_len = shared_prefix(&child.key, tokens, self.block_size);
 
             // We are done, the prefix is already in the trie.
-            if shared_prefix_len == tokens.len() {
+            if shared_prefix_len == tokens.len() || shared_prefix_len == 0 {
                 return Ok(shared_prefix_len);
             }
 
@@ -387,7 +414,7 @@ impl RadixTrie {
                     + self.insert_(
                         child_id,
                         &tokens[shared_prefix_len..],
-                        &blocks[shared_prefix_len..],
+                        &blocks[shared_prefix_len / self.block_size..],
                     )?);
             }
 
@@ -396,7 +423,7 @@ impl RadixTrie {
             // remainder of the prefix into the node again
             let child_id = self.split_node(child_id, shared_prefix_len);
             let key = &tokens[shared_prefix_len..];
-            let blocks = &blocks[shared_prefix_len..];
+            let blocks = &blocks[shared_prefix_len / self.block_size..];
             Ok(shared_prefix_len + self.insert_(child_id, key, blocks)?)
         } else {
             self.add_node(node_id, tokens, blocks);
@@ -550,34 +577,53 @@ impl TrieNode {
     }
 }
 
-/// Helper trait to get the length of the shared prefix of two sequences.
-trait SharedPrefixLen {
-    fn shared_prefix_len(&self, other: &Self) -> usize;
-}
-
-impl<T> SharedPrefixLen for [T]
-where
-    T: PartialEq,
-{
-    fn shared_prefix_len(&self, other: &Self) -> usize {
-        self.iter().zip(other).take_while(|(a, b)| a == b).count()
-    }
+fn shared_prefix(left: &[u32], right: &[u32], block_size: usize) -> usize {
+    let full = left.iter().zip(right).take_while(|(a, b)| a == b).count();
+    (full / block_size) * block_size
 }
 
 #[cfg(test)]
 mod tests {
     use std::sync::Arc;
 
-    use crate::block_allocator::Allocator;
+    use super::*;
 
-    use super::RadixAllocator;
+    #[test]
+    fn allocator_block_size() {
+        let mut cache = RadixAllocator::new(2, 12, None);
+        let allocation = cache.allocate(8, Some(Arc::new(vec![0, 1, 2, 3]))).unwrap();
+        assert_eq!(allocation.blocks, vec![8, 9, 10, 11]);
+        assert_eq!(allocation.slots, vec![16, 17, 18, 19, 20, 21, 22, 23]);
+        assert_eq!(allocation.prefix_len, 0);
+        cache.free(allocation.blocks.clone(), allocation.allocation_id);
+
+        let allocation = cache.allocate(8, Some(Arc::new(vec![0, 1, 2, 3]))).unwrap();
+        assert_eq!(allocation.blocks, vec![8, 9, 10, 11]);
+        assert_eq!(allocation.slots, vec![16, 17, 18, 19, 20, 21, 22, 23]);
+        assert_eq!(allocation.prefix_len, 4);
+    }
+
+    #[test]
+    fn allocator_block_size_non_aligned() {
+        let mut cache = RadixAllocator::new(2, 12, None);
+        let allocation = cache.allocate(7, Some(Arc::new(vec![0, 1, 2]))).unwrap();
+        assert_eq!(allocation.blocks, vec![8, 9, 10, 11]);
+        assert_eq!(allocation.slots, vec![16, 17, 18, 19, 20, 21, 22]);
+        assert_eq!(allocation.prefix_len, 0);
+        cache.free(allocation.blocks.clone(), allocation.allocation_id);
+
+        let allocation = cache.allocate(7, Some(Arc::new(vec![0, 1, 2]))).unwrap();
+        assert_eq!(allocation.blocks, vec![8, 9, 10, 11]);
+        assert_eq!(allocation.slots, vec![16, 17, 18, 19, 20, 21, 22]);
+        assert_eq!(allocation.prefix_len, 2);
+    }
 
     #[test]
     fn allocator_reuses_prefixes() {
         let mut cache = RadixAllocator::new(1, 12, None);
         let allocation = cache.allocate(8, Some(Arc::new(vec![0, 1, 2, 3]))).unwrap();
         assert_eq!(allocation.blocks, vec![4, 5, 6, 7, 8, 9, 10, 11]);
-        assert_eq!(allocation.slots, allocation.slots);
+        assert_eq!(allocation.blocks, allocation.slots);
         assert_eq!(allocation.prefix_len, 0);
         cache.free(allocation.blocks.clone(), allocation.allocation_id);
 
@@ -666,7 +712,7 @@ mod tests {
 
     #[test]
     fn trie_insertions_have_correct_prefix_len() {
-        let mut trie = super::RadixTrie::new();
+        let mut trie = RadixTrie::new(1);
 
         assert_eq!(trie.insert(&[0, 1, 2], &[0, 1, 2]).unwrap(), 0);
 
@@ -687,9 +733,33 @@ mod tests {
         );
     }
 
+    #[test]
+    fn trie_insertions_block_size() {
+        let mut trie = RadixTrie::new(2);
+
+        assert_eq!(trie.insert(&[0, 1, 2, 3], &[0, 1]).unwrap(), 0);
+
+        // Already exists.
+        // But needs to be block_size aligned
+        assert_eq!(trie.insert(&[0, 1, 2, 3], &[0, 1]).unwrap(), 4);
+
+        // Completely new at root-level
+        assert_eq!(trie.insert(&[1, 2, 3, 4], &[1, 2]).unwrap(), 0);
+
+        // Contains full prefix, but longer.
+        assert_eq!(trie.insert(&[0, 1, 2, 3, 4, 5], &[0, 1, 2]).unwrap(), 4);
+
+        // Shares partial prefix, we need a split.
+        assert_eq!(
+            trie.insert(&[0, 1, 3, 4, 5, 6, 7, 8], &[0, 1, 2, 3])
+                .unwrap(),
+            2
+        );
+    }
+
     #[test]
     fn trie_get_returns_correct_blocks() {
-        let mut trie = super::RadixTrie::new();
+        let mut trie = RadixTrie::new(1);
         trie.insert(&[0, 1, 2], &[0, 1, 2]).unwrap();
         trie.insert(&[1, 2, 3], &[1, 2, 3]).unwrap();
         trie.insert(&[0, 1, 2, 3, 4], &[0, 1, 2, 3, 4]).unwrap();
@@ -723,7 +793,7 @@ mod tests {
 
     #[test]
     fn trie_evict_removes_correct_blocks() {
-        let mut trie = super::RadixTrie::new();
+        let mut trie = RadixTrie::new(1);
         trie.insert(&[0, 1, 2], &[0, 1, 2]).unwrap();
         trie.insert(&[0, 1, 2, 3, 5, 6, 7], &[0, 1, 2, 3, 5, 6, 7])
             .unwrap();
diff --git a/benchmark/src/generation.rs b/benchmark/src/generation.rs
index 7494d5b5..789c7b51 100644
--- a/benchmark/src/generation.rs
+++ b/benchmark/src/generation.rs
@@ -148,6 +148,7 @@ async fn prefill(
             }),
             inputs: sequence.clone(),
             truncate: sequence_length,
+            add_special_tokens: true,
             parameters: Some(parameters.clone()),
             stopping_parameters: Some(StoppingCriteriaParameters {
                 max_new_tokens: decode_length,
diff --git a/flake.lock b/flake.lock
index 14011768..c0a696b1 100644
--- a/flake.lock
+++ b/flake.lock
@@ -835,11 +835,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1724206841,
-        "narHash": "sha256-L8dKaX4T3k+TR2fEHCfGbH4UXdspovz/pj87iai9qmc=",
+        "lastModified": 1724638882,
+        "narHash": "sha256-ap2jIQi/FuUHR6HCht6ASWhoz8EiB99XmI8Esot38VE=",
         "owner": "oxalica",
         "repo": "rust-overlay",
-        "rev": "45e98fbd62c32e5927e952d2833fa1ba4fb35a61",
+        "rev": "19b70f147b9c67a759e35824b241f1ed92e46694",
         "type": "github"
       },
       "original": {
diff --git a/integration-tests/models/__snapshots__/test_chat_llama/test_flash_llama_simple.json b/integration-tests/models/__snapshots__/test_chat_llama/test_flash_llama_simple.json
index 8631c076..5553e17d 100644
--- a/integration-tests/models/__snapshots__/test_chat_llama/test_flash_llama_simple.json
+++ b/integration-tests/models/__snapshots__/test_chat_llama/test_flash_llama_simple.json
@@ -5,7 +5,7 @@
       "index": 0,
       "logprobs": null,
       "message": {
-        "content": "As of your last question, the weather in Brooklyn, New York, is typically hot and humid throughout the year. The suburbs around New York City are jealously sheltered, and at least in the Lower Bronx, there are very few outdoor environments to explore in the middle of urban confines. In fact, typical times for humidity levels in Brooklyn include:\n\n- Early morning: 80-85% humidity, with occas",
+        "content": "As of your last question, the weather in Brooklyn, New York, is typically hot and humid throughout the year. The suburbs around New York City are jealously sheltered, and at least in the Lower Bronx, there are very few outdoor environments to appreciate nature.\n\nIn terms of temperature, the warmest times of the year are from June to August, when average high temperatures typically range from around 73°F or 23°C",
         "name": null,
         "role": "assistant",
         "tool_calls": null
@@ -13,14 +13,14 @@
       "usage": null
     }
   ],
-  "created": 1716553098,
+  "created": 1724792495,
   "id": "",
   "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
-  "object": "text_completion",
-  "system_fingerprint": "2.0.5-dev0-native",
+  "object": "chat.completion",
+  "system_fingerprint": "2.2.1-dev0-native",
   "usage": {
     "completion_tokens": 100,
-    "prompt_tokens": 62,
-    "total_tokens": 162
+    "prompt_tokens": 61,
+    "total_tokens": 161
   }
 }
diff --git a/integration-tests/models/__snapshots__/test_completion_prompts/test_flash_llama_completion_many_prompts_stream.json b/integration-tests/models/__snapshots__/test_completion_prompts/test_flash_llama_completion_many_prompts_stream.json
index d87071cf..e7fb5740 100644
--- a/integration-tests/models/__snapshots__/test_completion_prompts/test_flash_llama_completion_many_prompts_stream.json
+++ b/integration-tests/models/__snapshots__/test_completion_prompts/test_flash_llama_completion_many_prompts_stream.json
@@ -8,11 +8,11 @@
         "text": "\n"
       }
     ],
-    "created": 1713284431,
+    "created": 1724833943,
     "id": "",
     "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
@@ -23,11 +23,11 @@
         "text": "\n"
       }
     ],
-    "created": 1713284431,
+    "created": 1724833943,
     "id": "",
     "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
@@ -38,11 +38,11 @@
         "text": "\n"
       }
     ],
-    "created": 1713284431,
+    "created": 1724833943,
     "id": "",
     "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
@@ -53,11 +53,11 @@
         "text": "hd"
       }
     ],
-    "created": 1713284431,
+    "created": 1724833943,
     "id": "",
     "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
@@ -68,11 +68,11 @@
         "text": "\n"
       }
     ],
-    "created": 1713284431,
+    "created": 1724833943,
     "id": "",
     "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
@@ -83,11 +83,11 @@
         "text": "\n"
       }
     ],
-    "created": 1713284431,
+    "created": 1724833943,
     "id": "",
     "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
@@ -98,11 +98,11 @@
         "text": "\n"
       }
     ],
-    "created": 1713284431,
+    "created": 1724833943,
     "id": "",
     "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
@@ -113,11 +113,11 @@
         "text": "aho"
       }
     ],
-    "created": 1713284431,
+    "created": 1724833943,
     "id": "",
     "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
@@ -128,11 +128,11 @@
         "text": "2"
       }
     ],
-    "created": 1713284431,
+    "created": 1724833943,
     "id": "",
     "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
@@ -143,11 +143,11 @@
         "text": "2"
       }
     ],
-    "created": 1713284431,
+    "created": 1724833943,
     "id": "",
     "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
@@ -158,11 +158,11 @@
         "text": "2"
       }
     ],
-    "created": 1713284431,
+    "created": 1724833943,
     "id": "",
     "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
@@ -173,11 +173,11 @@
         "text": "ima"
       }
     ],
-    "created": 1713284431,
+    "created": 1724833943,
     "id": "",
     "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
@@ -188,11 +188,11 @@
         "text": "."
       }
     ],
-    "created": 1713284431,
+    "created": 1724833943,
     "id": "",
     "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
@@ -203,11 +203,11 @@
         "text": "."
       }
     ],
-    "created": 1713284431,
+    "created": 1724833943,
     "id": "",
     "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
@@ -218,11 +218,11 @@
         "text": "."
       }
     ],
-    "created": 1713284431,
+    "created": 1724833943,
     "id": "",
     "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
@@ -233,11 +233,11 @@
         "text": "\n"
       }
     ],
-    "created": 1713284431,
+    "created": 1724833943,
     "id": "",
     "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
@@ -248,11 +248,11 @@
         "text": " Sarah"
       }
     ],
-    "created": 1713284431,
+    "created": 1724833943,
     "id": "",
     "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
@@ -263,11 +263,11 @@
         "text": " Yes"
       }
     ],
-    "created": 1713284431,
+    "created": 1724833943,
     "id": "",
     "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
@@ -278,11 +278,11 @@
         "text": " And"
       }
     ],
-    "created": 1713284431,
+    "created": 1724833943,
     "id": "",
     "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
@@ -293,11 +293,11 @@
         "text": "i"
       }
     ],
-    "created": 1713284431,
+    "created": 1724833943,
     "id": "",
     "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
@@ -308,11 +308,11 @@
         "text": "'"
       }
     ],
-    "created": 1713284431,
+    "created": 1724833943,
     "id": "",
     "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
@@ -323,11 +323,11 @@
         "text": ","
       }
     ],
-    "created": 1713284431,
+    "created": 1724833943,
     "id": "",
     "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
@@ -338,11 +338,11 @@
         "text": " what"
       }
     ],
-    "created": 1713284431,
+    "created": 1724833943,
     "id": "",
     "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
@@ -353,11 +353,11 @@
         "text": "'"
       }
     ],
-    "created": 1713284431,
+    "created": 1724833943,
     "id": "",
     "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
@@ -368,11 +368,11 @@
         "text": "s"
       }
     ],
-    "created": 1713284431,
+    "created": 1724833943,
     "id": "",
     "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
@@ -383,11 +383,11 @@
         "text": " Moh"
       }
     ],
-    "created": 1713284431,
+    "created": 1724833943,
     "id": "",
     "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
@@ -398,11 +398,11 @@
         "text": " is"
       }
     ],
-    "created": 1713284431,
+    "created": 1724833943,
     "id": "",
     "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
@@ -413,11 +413,11 @@
         "text": "m"
       }
     ],
-    "created": 1713284431,
+    "created": 1724833943,
     "id": "",
     "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
@@ -428,11 +428,11 @@
         "text": " Room"
       }
     ],
-    "created": 1713284431,
+    "created": 1724833943,
     "id": "",
     "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
@@ -443,11 +443,11 @@
         "text": "s"
       }
     ],
-    "created": 1713284431,
+    "created": 1724833943,
     "id": "",
     "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
@@ -458,11 +458,11 @@
         "text": " the"
       }
     ],
-    "created": 1713284431,
+    "created": 1724833943,
     "id": "",
     "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
@@ -473,11 +473,11 @@
         "text": " tired"
       }
     ],
-    "created": 1713284431,
+    "created": 1724833943,
     "id": "",
     "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
@@ -488,11 +488,11 @@
         "text": ":"
       }
     ],
-    "created": 1713284431,
+    "created": 1724833943,
     "id": "",
     "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
@@ -503,11 +503,11 @@
         "text": "'"
       }
     ],
-    "created": 1713284431,
+    "created": 1724833943,
     "id": "",
     "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
@@ -518,11 +518,11 @@
         "text": " capital"
       }
     ],
-    "created": 1713284431,
+    "created": 1724833943,
     "id": "",
     "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
@@ -530,73 +530,73 @@
         "finish_reason": "",
         "index": 3,
         "logprobs": null,
-        "text": " of"
+        "text": ","
       }
     ],
-    "created": 1713284431,
+    "created": 1724833943,
     "id": "",
     "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
       {
-        "finish_reason": "",
+        "finish_reason": "length",
         "index": 0,
         "logprobs": null,
         "text": " She"
       }
     ],
-    "created": 1713284431,
+    "created": 1724833943,
     "id": "",
     "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
       {
-        "finish_reason": "",
+        "finish_reason": "length",
         "index": 1,
         "logprobs": null,
         "text": " scale"
       }
     ],
-    "created": 1713284431,
+    "created": 1724833943,
     "id": "",
     "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
       {
-        "finish_reason": "",
+        "finish_reason": "length",
         "index": 2,
         "logprobs": null,
         "text": " of"
       }
     ],
-    "created": 1713284431,
+    "created": 1724833943,
     "id": "",
     "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   },
   {
     "choices": [
       {
-        "finish_reason": "",
+        "finish_reason": "length",
         "index": 3,
         "logprobs": null,
-        "text": " being"
+        "text": " its"
       }
     ],
-    "created": 1713284431,
+    "created": 1724833943,
     "id": "",
     "model": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
     "object": "text_completion",
-    "system_fingerprint": "2.0.1-native"
+    "system_fingerprint": "2.2.1-dev0-native"
   }
 ]
diff --git a/integration-tests/models/__snapshots__/test_flash_deepseek_v2/test_flash_deepseek_v2.json b/integration-tests/models/__snapshots__/test_flash_deepseek_v2/test_flash_deepseek_v2.json
index 03f90367..732b0c49 100644
--- a/integration-tests/models/__snapshots__/test_flash_deepseek_v2/test_flash_deepseek_v2.json
+++ b/integration-tests/models/__snapshots__/test_flash_deepseek_v2/test_flash_deepseek_v2.json
@@ -16,7 +16,7 @@
       },
       {
         "id": 3102,
-        "logprob": -11.1875,
+        "logprob": -11.25,
         "text": " request"
       }
     ],
@@ -24,66 +24,66 @@
     "tokens": [
       {
         "id": 185,
-        "logprob": -1.5546875,
+        "logprob": -1.546875,
         "special": false,
         "text": "\n"
       },
       {
         "id": 549,
-        "logprob": -2.84375,
+        "logprob": -2.859375,
         "special": false,
         "text": "The"
       },
       {
         "id": 1727,
-        "logprob": -2.34375,
+        "logprob": -2.484375,
         "special": false,
         "text": " test"
       },
       {
         "id": 3102,
-        "logprob": -0.8359375,
+        "logprob": -0.83203125,
         "special": false,
         "text": " request"
       },
       {
         "id": 317,
-        "logprob": -1.0859375,
+        "logprob": -1.1484375,
         "special": false,
         "text": " is"
       },
       {
-        "id": 254,
-        "logprob": -1.5390625,
+        "id": 245,
+        "logprob": -1.578125,
         "special": false,
-        "text": " the"
+        "text": " a"
       },
       {
-        "id": 1022,
-        "logprob": -1.1875,
+        "id": 3412,
+        "logprob": -2.578125,
         "special": false,
-        "text": " first"
+        "text": " document"
       },
       {
-        "id": 3458,
-        "logprob": -0.35546875,
+        "id": 344,
+        "logprob": -1.125,
         "special": false,
-        "text": " step"
+        "text": " that"
       },
       {
-        "id": 279,
-        "logprob": -0.8828125,
+        "id": 317,
+        "logprob": -1.6953125,
         "special": false,
-        "text": " in"
+        "text": " is"
       },
       {
-        "id": 254,
-        "logprob": -0.71484375,
+        "id": 1222,
+        "logprob": -1.71875,
         "special": false,
-        "text": " the"
+        "text": " used"
       }
     ],
     "top_tokens": null
   },
-  "generated_text": "\nThe test request is the first step in the"
+  "generated_text": "\nThe test request is a document that is used"
 }
diff --git a/integration-tests/models/__snapshots__/test_flash_deepseek_v2/test_flash_deepseek_v2_load.json b/integration-tests/models/__snapshots__/test_flash_deepseek_v2/test_flash_deepseek_v2_load.json
index e365829a..f1eeab25 100644
--- a/integration-tests/models/__snapshots__/test_flash_deepseek_v2/test_flash_deepseek_v2_load.json
+++ b/integration-tests/models/__snapshots__/test_flash_deepseek_v2/test_flash_deepseek_v2_load.json
@@ -37,56 +37,56 @@
         },
         {
           "id": 1727,
-          "logprob": -2.359375,
+          "logprob": -2.4375,
           "special": false,
           "text": " test"
         },
         {
           "id": 3102,
-          "logprob": -0.83203125,
+          "logprob": -0.83984375,
           "special": false,
           "text": " request"
         },
         {
           "id": 317,
-          "logprob": -1.125,
+          "logprob": -1.1328125,
           "special": false,
           "text": " is"
         },
         {
-          "id": 245,
-          "logprob": -1.5703125,
+          "id": 254,
+          "logprob": -1.515625,
           "special": false,
-          "text": " a"
+          "text": " the"
         },
         {
-          "id": 3412,
-          "logprob": -2.578125,
+          "id": 1022,
+          "logprob": -1.15625,
           "special": false,
-          "text": " document"
+          "text": " first"
         },
         {
-          "id": 344,
-          "logprob": -1.125,
+          "id": 3458,
+          "logprob": -0.3671875,
           "special": false,
-          "text": " that"
+          "text": " step"
         },
         {
-          "id": 317,
-          "logprob": -1.6953125,
+          "id": 279,
+          "logprob": -0.88671875,
           "special": false,
-          "text": " is"
+          "text": " in"
         },
         {
-          "id": 1222,
-          "logprob": -1.75,
+          "id": 254,
+          "logprob": -0.69140625,
           "special": false,
-          "text": " used"
+          "text": " the"
         }
       ],
       "top_tokens": null
     },
-    "generated_text": "\nThe test request is a document that is used"
+    "generated_text": "\nThe test request is the first step in the"
   },
   {
     "details": {
@@ -126,56 +126,56 @@
         },
         {
           "id": 1727,
-          "logprob": -2.359375,
+          "logprob": -2.4375,
           "special": false,
           "text": " test"
         },
         {
           "id": 3102,
-          "logprob": -0.83203125,
+          "logprob": -0.83984375,
           "special": false,
           "text": " request"
         },
         {
           "id": 317,
-          "logprob": -1.125,
+          "logprob": -1.1328125,
           "special": false,
           "text": " is"
         },
         {
-          "id": 245,
-          "logprob": -1.5703125,
+          "id": 254,
+          "logprob": -1.515625,
           "special": false,
-          "text": " a"
+          "text": " the"
         },
         {
-          "id": 3412,
-          "logprob": -2.578125,
+          "id": 1022,
+          "logprob": -1.15625,
           "special": false,
-          "text": " document"
+          "text": " first"
         },
         {
-          "id": 344,
-          "logprob": -1.125,
+          "id": 3458,
+          "logprob": -0.3671875,
           "special": false,
-          "text": " that"
+          "text": " step"
         },
         {
-          "id": 317,
-          "logprob": -1.6953125,
+          "id": 279,
+          "logprob": -0.88671875,
           "special": false,
-          "text": " is"
+          "text": " in"
         },
         {
-          "id": 1222,
-          "logprob": -1.75,
+          "id": 254,
+          "logprob": -0.69140625,
           "special": false,
-          "text": " used"
+          "text": " the"
         }
       ],
       "top_tokens": null
     },
-    "generated_text": "\nThe test request is a document that is used"
+    "generated_text": "\nThe test request is the first step in the"
   },
   {
     "details": {
@@ -215,56 +215,56 @@
         },
         {
           "id": 1727,
-          "logprob": -2.359375,
+          "logprob": -2.4375,
           "special": false,
           "text": " test"
         },
         {
           "id": 3102,
-          "logprob": -0.83203125,
+          "logprob": -0.83984375,
           "special": false,
           "text": " request"
         },
         {
           "id": 317,
-          "logprob": -1.125,
+          "logprob": -1.1328125,
           "special": false,
           "text": " is"
         },
         {
-          "id": 245,
-          "logprob": -1.5703125,
+          "id": 254,
+          "logprob": -1.515625,
           "special": false,
-          "text": " a"
+          "text": " the"
         },
         {
-          "id": 3412,
-          "logprob": -2.578125,
+          "id": 1022,
+          "logprob": -1.15625,
           "special": false,
-          "text": " document"
+          "text": " first"
         },
         {
-          "id": 344,
-          "logprob": -1.125,
+          "id": 3458,
+          "logprob": -0.3671875,
           "special": false,
-          "text": " that"
+          "text": " step"
         },
         {
-          "id": 317,
-          "logprob": -1.6953125,
+          "id": 279,
+          "logprob": -0.88671875,
           "special": false,
-          "text": " is"
+          "text": " in"
         },
         {
-          "id": 1222,
-          "logprob": -1.75,
+          "id": 254,
+          "logprob": -0.69140625,
           "special": false,
-          "text": " used"
+          "text": " the"
         }
       ],
       "top_tokens": null
     },
-    "generated_text": "\nThe test request is a document that is used"
+    "generated_text": "\nThe test request is the first step in the"
   },
   {
     "details": {
@@ -304,55 +304,55 @@
         },
         {
           "id": 1727,
-          "logprob": -2.359375,
+          "logprob": -2.4375,
           "special": false,
           "text": " test"
         },
         {
           "id": 3102,
-          "logprob": -0.83203125,
+          "logprob": -0.83984375,
           "special": false,
           "text": " request"
         },
         {
           "id": 317,
-          "logprob": -1.125,
+          "logprob": -1.1328125,
           "special": false,
           "text": " is"
         },
         {
-          "id": 245,
-          "logprob": -1.5703125,
+          "id": 254,
+          "logprob": -1.515625,
           "special": false,
-          "text": " a"
+          "text": " the"
         },
         {
-          "id": 3412,
-          "logprob": -2.578125,
+          "id": 1022,
+          "logprob": -1.15625,
           "special": false,
-          "text": " document"
+          "text": " first"
         },
         {
-          "id": 344,
-          "logprob": -1.125,
+          "id": 3458,
+          "logprob": -0.3671875,
           "special": false,
-          "text": " that"
+          "text": " step"
         },
         {
-          "id": 317,
-          "logprob": -1.6953125,
+          "id": 279,
+          "logprob": -0.88671875,
           "special": false,
-          "text": " is"
+          "text": " in"
         },
         {
-          "id": 1222,
-          "logprob": -1.75,
+          "id": 254,
+          "logprob": -0.69140625,
           "special": false,
-          "text": " used"
+          "text": " the"
         }
       ],
       "top_tokens": null
     },
-    "generated_text": "\nThe test request is a document that is used"
+    "generated_text": "\nThe test request is the first step in the"
   }
 ]
diff --git a/integration-tests/models/__snapshots__/test_flash_llama_fp8/test_flash_llama_fp8_all_params.json b/integration-tests/models/__snapshots__/test_flash_llama_fp8/test_flash_llama_fp8_all_params.json
index bf981e4f..e39829ec 100644
--- a/integration-tests/models/__snapshots__/test_flash_llama_fp8/test_flash_llama_fp8_all_params.json
+++ b/integration-tests/models/__snapshots__/test_flash_llama_fp8/test_flash_llama_fp8_all_params.json
@@ -1,8 +1,8 @@
 {
   "details": {
     "best_of_sequences": null,
-    "finish_reason": "length",
-    "generated_tokens": 10,
+    "finish_reason": "stop_sequence",
+    "generated_tokens": 5,
     "prefill": [
       {
         "id": 128000,
@@ -16,7 +16,7 @@
       },
       {
         "id": 1715,
-        "logprob": -10.375,
+        "logprob": -10.4375,
         "text": " request"
       }
     ],
@@ -29,61 +29,31 @@
         "text": ":"
       },
       {
-        "id": 2209,
-        "logprob": -2.78125,
+        "id": 923,
+        "logprob": -2.84375,
         "special": false,
-        "text": " Is"
+        "text": " add"
       },
       {
-        "id": 279,
-        "logprob": -0.6328125,
+        "id": 264,
+        "logprob": 0.0,
         "special": false,
-        "text": " the"
-      },
-      {
-        "id": 734,
-        "logprob": -2.703125,
-        "special": false,
-        "text": " function"
+        "text": " a"
       },
       {
         "id": 330,
-        "logprob": -0.34179688,
+        "logprob": -0.31640625,
         "special": false,
         "text": " \""
       },
       {
-        "id": 4110,
-        "logprob": -2.359375,
+        "id": 1985,
+        "logprob": 0.0,
         "special": false,
-        "text": "Create"
-      },
-      {
-        "id": 7575,
-        "logprob": -2.1875,
-        "special": false,
-        "text": "Process"
-      },
-      {
-        "id": 1,
-        "logprob": -0.07910156,
-        "special": false,
-        "text": "\""
-      },
-      {
-        "id": 304,
-        "logprob": -0.83203125,
-        "special": false,
-        "text": " in"
-      },
-      {
-        "id": 12468,
-        "logprob": -1.8203125,
-        "special": false,
-        "text": " Win"
+        "text": "test"
       }
     ],
     "top_tokens": null
   },
-  "generated_text": "Test request: Is the function \"CreateProcess\" in Win"
+  "generated_text": "Test request: add a \"test"
 }
diff --git a/integration-tests/models/__snapshots__/test_flash_starcoder2/test_flash_starcoder2_default_params.json b/integration-tests/models/__snapshots__/test_flash_starcoder2/test_flash_starcoder2_default_params.json
index d882b82a..412b19b4 100644
--- a/integration-tests/models/__snapshots__/test_flash_starcoder2/test_flash_starcoder2_default_params.json
+++ b/integration-tests/models/__snapshots__/test_flash_starcoder2/test_flash_starcoder2_default_params.json
@@ -16,7 +16,7 @@
       },
       {
         "id": 100,
-        "logprob": -0.38549805,
+        "logprob": -0.38305664,
         "text": "_"
       },
       {
@@ -29,7 +29,7 @@
     "tokens": [
       {
         "id": 2284,
-        "logprob": -0.31323242,
+        "logprob": -0.296875,
         "special": false,
         "text": "():"
       },
@@ -59,19 +59,19 @@
       },
       {
         "id": 10914,
-        "logprob": -0.7817383,
+        "logprob": -0.7734375,
         "special": false,
         "text": " World"
       },
       {
         "id": 16013,
-        "logprob": -0.6328125,
+        "logprob": -0.61816406,
         "special": false,
         "text": "!\")"
       },
       {
         "id": 222,
-        "logprob": -0.0619812,
+        "logprob": -0.054870605,
         "special": false,
         "text": "\n"
       },
@@ -83,7 +83,7 @@
       },
       {
         "id": 610,
-        "logprob": -0.4086914,
+        "logprob": -0.4152832,
         "special": false,
         "text": "def"
       },
@@ -113,7 +113,7 @@
       },
       {
         "id": 444,
-        "logprob": -0.21826172,
+        "logprob": -0.21618652,
         "special": false,
         "text": "name"
       },
@@ -173,7 +173,7 @@
       },
       {
         "id": 11571,
-        "logprob": -0.10021973,
+        "logprob": -0.08892822,
         "special": false,
         "text": "!\""
       },
diff --git a/integration-tests/models/__snapshots__/test_idefics2/test_flash_idefics2_next_all_params.json b/integration-tests/models/__snapshots__/test_idefics2/test_flash_idefics2_next_all_params.json
index 1fad0b96..dab437b9 100644
--- a/integration-tests/models/__snapshots__/test_idefics2/test_flash_idefics2_next_all_params.json
+++ b/integration-tests/models/__snapshots__/test_idefics2/test_flash_idefics2_next_all_params.json
@@ -30,19 +30,19 @@
       },
       {
         "id": 264,
-        "logprob": -0.37573242,
+        "logprob": -0.38061523,
         "special": false,
         "text": " a"
       },
       {
         "id": 633,
-        "logprob": -0.09161377,
+        "logprob": -0.09301758,
         "special": false,
         "text": " new"
       },
       {
         "id": 4480,
-        "logprob": -0.26171875,
+        "logprob": -0.26782227,
         "special": false,
         "text": " feature"
       },
@@ -78,7 +78,7 @@
       },
       {
         "id": 13,
-        "logprob": 0.0,
+        "logprob": -0.10632324,
         "special": false,
         "text": "\n"
       }
diff --git a/integration-tests/models/test_chat_llama.py b/integration-tests/models/test_chat_llama.py
index 1f7a4a59..7d24add3 100644
--- a/integration-tests/models/test_chat_llama.py
+++ b/integration-tests/models/test_chat_llama.py
@@ -35,6 +35,6 @@ async def test_flash_llama_simple(flash_llama_chat, response_snapshot):
     print(repr(response.choices[0].message.content))
     assert (
         response.choices[0].message.content
-        == "As of your last question, the weather in Brooklyn, New York, is typically hot and humid throughout the year. The suburbs around New York City are jealously sheltered, and at least in the Lower Bronx, there are very few outdoor environments to explore in the middle of urban confines. In fact, typical times for humidity levels in Brooklyn include:\n\n- Early morning: 80-85% humidity, with occas"
+        == "As of your last question, the weather in Brooklyn, New York, is typically hot and humid throughout the year. The suburbs around New York City are jealously sheltered, and at least in the Lower Bronx, there are very few outdoor environments to appreciate nature.\n\nIn terms of temperature, the warmest times of the year are from June to August, when average high temperatures typically range from around 73°F or 23°C"
     )
     assert response == response_snapshot
diff --git a/launcher/src/main.rs b/launcher/src/main.rs
index 9a90a673..8e5c9dcd 100644
--- a/launcher/src/main.rs
+++ b/launcher/src/main.rs
@@ -8,7 +8,7 @@ use nix::unistd::Pid;
 use serde::Deserialize;
 use std::env;
 use std::ffi::OsString;
-use std::io::{BufRead, BufReader, Lines};
+use std::io::{BufRead, BufReader};
 use std::os::unix::process::{CommandExt, ExitStatusExt};
 use std::path::Path;
 use std::process::{Child, Command, ExitStatus, Stdio};
@@ -18,12 +18,103 @@ use std::sync::{mpsc, Arc};
 use std::thread;
 use std::thread::sleep;
 use std::time::{Duration, Instant};
-use std::{fs, io};
+use std::{
+    fs, io,
+    io::{Read, Write},
+};
 use thiserror::Error;
 use tracing_subscriber::{filter::LevelFilter, EnvFilter};
 
 mod env_runtime;
 
+fn get_config(
+    model_id: &str,
+    revision: &Option<String>,
+) -> Result<Config, Box<dyn std::error::Error>> {
+    let mut path = std::path::Path::new(model_id).to_path_buf();
+    let model_id = model_id.to_string();
+    let filename = if !path.exists() {
+        // Assume it's a hub id
+
+        let api = if let Ok(token) = std::env::var("HF_TOKEN") {
+            // env variable has precedence over on file token.
+            ApiBuilder::new().with_token(Some(token)).build()?
+        } else {
+            Api::new()?
+        };
+        let repo = if let Some(ref revision) = revision {
+            api.repo(Repo::with_revision(
+                model_id,
+                RepoType::Model,
+                revision.to_string(),
+            ))
+        } else {
+            api.model(model_id)
+        };
+        repo.get("config.json")?
+    } else {
+        path.push("config.json");
+        path
+    };
+
+    let content = std::fs::read_to_string(filename)?;
+    let config: RawConfig = serde_json::from_str(&content)?;
+
+    let config: Config = config.into();
+    Ok(config)
+}
+
+fn resolve_attention(config: &Option<Config>, lora_adapters: &Option<String>) -> (String, String) {
+    let mut prefix_caching: Option<String> = std::env::var("USE_PREFIX_CACHING").ok();
+    let mut attention: Option<String> = std::env::var("ATTENTION").ok();
+    if let Some(config) = config {
+        if prefix_caching.is_none() {
+            if config.vision_config.is_some() {
+                tracing::info!("Disabling prefix caching because of VLM model");
+                prefix_caching = Some("0".to_string());
+            } else if config.is_encoder_decoder {
+                tracing::info!("Disabling prefix caching because of seq2seq model");
+                prefix_caching = Some("0".to_string());
+            }
+        }
+        match config.head_dim {
+            Some(h) if h == 64 || h == 128 || h == 256 => {
+                if lora_adapters.is_some() && prefix_caching.is_none() {
+                    tracing::info!("Disabling prefix caching because of lora adapters");
+                    prefix_caching = Some("0".to_string());
+                }
+                match config.model_type.as_deref() {
+                    Some("gemma2") | Some("falcon") | Some("deepseek_v2") => {
+                        // Required because gemma2 needs bfloat16 which is not supported by
+                        // flashinfer ?
+                        if attention.is_none() {
+                            tracing::info!(
+                                "Forcing flash decoding because model {} requires it",
+                                config.model_type.as_ref().unwrap()
+                            );
+                            attention = Some("flashdecoding".to_string());
+                        }
+                    }
+                    Some("t5") => {}
+                    _ => {}
+                }
+            }
+            _ => {
+                if attention.is_none() {
+                    tracing::info!("Forcing flash decoding because head dim is not supported by flashinfer, also disabling prefix caching");
+                    attention = Some("flashdecoding".to_string());
+                }
+                if prefix_caching.is_none() {
+                    prefix_caching = Some("0".to_string());
+                }
+            }
+        }
+    }
+    let prefix_caching = prefix_caching.unwrap_or("true".to_string());
+    let attention = attention.unwrap_or("flashinfer".to_string());
+    (prefix_caching, attention)
+}
+
 #[derive(Deserialize)]
 struct RawConfig {
     max_position_embeddings: Option<usize>,
@@ -31,6 +122,12 @@ struct RawConfig {
     model_type: Option<String>,
     max_seq_len: Option<usize>,
     quantization_config: Option<QuantizationConfig>,
+    n_embd: Option<usize>,
+    hidden_size: Option<usize>,
+    num_attention_heads: Option<usize>,
+    head_dim: Option<usize>,
+    vision_config: Option<VisionConfig>,
+    is_encoder_decoder: Option<bool>,
 }
 
 #[derive(Deserialize)]
@@ -38,10 +135,17 @@ struct QuantizationConfig {
     quant_method: Option<Quantization>,
 }
 
+#[derive(Deserialize)]
+struct VisionConfig {}
+
 #[derive(Deserialize)]
 struct Config {
     max_position_embeddings: Option<usize>,
     quantize: Option<Quantization>,
+    head_dim: Option<usize>,
+    model_type: Option<String>,
+    vision_config: Option<VisionConfig>,
+    is_encoder_decoder: bool,
 }
 
 impl From<RawConfig> for Config {
@@ -51,9 +155,32 @@ impl From<RawConfig> for Config {
             .or(other.max_seq_len)
             .or(other.n_positions);
         let quantize = other.quantization_config.and_then(|q| q.quant_method);
+        let head_dim = other.head_dim.or_else(|| {
+            match (other.hidden_size, other.n_embd, other.num_attention_heads) {
+                (Some(hidden_size), _, Some(num_attention_heads))
+                    if hidden_size % num_attention_heads == 0 =>
+                {
+                    Some(hidden_size / num_attention_heads)
+                }
+                // Legacy
+                (_, Some(hidden_size), Some(num_attention_heads))
+                    if hidden_size % num_attention_heads == 0 =>
+                {
+                    Some(hidden_size / num_attention_heads)
+                }
+                _ => None,
+            }
+        });
+        let model_type = other.model_type;
+        let vision_config = other.vision_config;
+        let is_encoder_decoder = other.is_encoder_decoder.unwrap_or(false);
         Config {
             max_position_embeddings,
             quantize,
+            head_dim,
+            model_type,
+            vision_config,
+            is_encoder_decoder,
         }
     }
 }
@@ -731,6 +858,7 @@ fn shard_manager(
         .args(shard_args)
         .env_clear()
         .envs(envs)
+        .stdin(Stdio::piped())
         .stdout(Stdio::piped())
         .stderr(Stdio::piped())
         .process_group(0)
@@ -752,12 +880,13 @@ fn shard_manager(
     };
 
     // Redirect STDOUT to the console
+    let mut pstdin = p.stdin.take().unwrap();
     let shard_stdout_reader = BufReader::new(p.stdout.take().unwrap());
     let shard_stderr_reader = BufReader::new(p.stderr.take().unwrap());
 
     //stdout tracing thread
     thread::spawn(move || {
-        log_lines(shard_stdout_reader.lines());
+        log_lines(shard_stdout_reader);
     });
     // We read stderr in another thread as it seems that lines() can block in some cases
     let (err_sender, err_receiver) = mpsc::channel();
@@ -766,6 +895,18 @@ fn shard_manager(
             err_sender.send(line).unwrap_or(());
         }
     });
+    // We read stdin in another thread as it seems that lines() can block in some cases
+    thread::spawn(move || {
+        let mut stdin = io::stdin(); // We get `Stdin` here.
+        loop {
+            let mut buffer = vec![0; 4096];
+            if let Ok(n) = stdin.read(&mut buffer) {
+                if n > 0 {
+                    let _ = pstdin.write_all(&buffer[..n]);
+                }
+            }
+        }
+    });
 
     let mut ready = false;
     let start_time = Instant::now();
@@ -872,19 +1013,36 @@ impl PythonLogMessage {
     }
 }
 
-impl TryFrom<&String> for PythonLogMessage {
+impl TryFrom<&[u8]> for PythonLogMessage {
     type Error = serde_json::Error;
 
-    fn try_from(value: &String) -> Result<Self, Self::Error> {
-        serde_json::from_str::<Self>(value)
+    fn try_from(value: &[u8]) -> Result<Self, Self::Error> {
+        serde_json::from_slice::<Self>(value)
     }
 }
 
-fn log_lines<S: Sized + BufRead>(lines: Lines<S>) {
-    for line in lines.map_while(Result::ok) {
-        match PythonLogMessage::try_from(&line) {
-            Ok(log) => log.trace(),
-            Err(_) => tracing::debug!("{line}"),
+fn log_lines<R: Sized + Read>(mut bufread: BufReader<R>) {
+    let mut buffer = vec![0u8; 8 * 4096];
+    let mut stdout = std::io::stdout();
+    loop {
+        let n = bufread.read(&mut buffer);
+        if let Ok(n) = n {
+            if n > 0 {
+                let mut lines = buffer[..n].split(|i| *i == b'\n').peekable();
+                while let Some(line) = lines.next() {
+                    match PythonLogMessage::try_from(line) {
+                        Ok(log) => log.trace(),
+                        // For interactive debugging ?
+                        Err(_) => {
+                            stdout.write_all(line).unwrap();
+                            if lines.peek().is_some() {
+                                stdout.write_all(b"\n").unwrap();
+                            }
+                            stdout.flush().unwrap();
+                        }
+                    }
+                }
+            }
         }
     }
 }
@@ -1044,7 +1202,7 @@ fn download_convert_model(
     let download_stdout = BufReader::new(download_process.stdout.take().unwrap());
 
     thread::spawn(move || {
-        log_lines(download_stdout.lines());
+        log_lines(download_stdout);
     });
 
     let download_stderr = BufReader::new(download_process.stderr.take().unwrap());
@@ -1439,68 +1597,35 @@ fn main() -> Result<(), LauncherError> {
 
     tracing::info!("{:#?}", args);
 
-    let get_max_positions_quantize =
-        || -> Result<(usize, Option<Quantization>), Box<dyn std::error::Error>> {
-            let model_id = args.model_id.clone();
-            let mut path = std::path::Path::new(&args.model_id).to_path_buf();
-            let filename = if !path.exists() {
-                // Assume it's a hub id
+    let config: Option<Config> = get_config(&args.model_id, &args.revision).ok();
+    let quantize = config.as_ref().and_then(|c| c.quantize);
+    // Quantization usually means you're even more RAM constrained.
+    let max_default = 4096;
 
-                let api = if let Ok(token) = std::env::var("HF_TOKEN") {
-                    // env variable has precedence over on file token.
-                    ApiBuilder::new().with_token(Some(token)).build()?
-                } else {
-                    Api::new()?
-                };
-                let repo = if let Some(ref revision) = args.revision {
-                    api.repo(Repo::with_revision(
-                        model_id,
-                        RepoType::Model,
-                        revision.to_string(),
-                    ))
-                } else {
-                    api.model(model_id)
-                };
-                repo.get("config.json")?
-            } else {
-                path.push("config.json");
-                path
-            };
-
-            let content = std::fs::read_to_string(filename)?;
-            let config: RawConfig = serde_json::from_str(&content)?;
-
-            if config.model_type == Some("gemma2".to_string()) {
-                tracing::info!("Forcing flash decoding because of softcap usage");
-                std::env::set_var("ATTENTION", "flashdecoding");
-            }
-            let config: Config = config.into();
-            let quantize = config.quantize;
-
-            // Quantization usually means you're even more RAM constrained.
-            let max_default = 4096;
-
-            if let Some(max_position_embeddings) = config.max_position_embeddings {
-                if max_position_embeddings > max_default {
-                    let max = max_position_embeddings;
-                    if args.max_input_tokens.is_none()
-                        && args.max_total_tokens.is_none()
-                        && args.max_batch_prefill_tokens.is_none()
-                    {
-                        tracing::info!("Model supports up to {max} but tgi will now set its default to {max_default} instead. This is to save VRAM by refusing large prompts in order to allow more users on the same hardware. You can increase that size using `--max-batch-prefill-tokens={} --max-total-tokens={max} --max-input-tokens={}`.", max + 50, max - 1);
-                    }
-                    Ok((max_default, quantize))
-                } else {
-                    Ok((max_position_embeddings, quantize))
+    let max_position_embeddings = if let Some(config) = &config {
+        if let Some(max_position_embeddings) = config.max_position_embeddings {
+            if max_position_embeddings > max_default {
+                let max = max_position_embeddings;
+                if args.max_input_tokens.is_none()
+                    && args.max_total_tokens.is_none()
+                    && args.max_batch_prefill_tokens.is_none()
+                {
+                    tracing::info!("Model supports up to {max} but tgi will now set its default to {max_default} instead. This is to save VRAM by refusing large prompts in order to allow more users on the same hardware. You can increase that size using `--max-batch-prefill-tokens={} --max-total-tokens={max} --max-input-tokens={}`.", max + 50, max - 1);
                 }
+                max_default
             } else {
-                Err(Box::new(LauncherError::ArgumentValidation(
-                    "no max defined".to_string(),
-                )))
+                max_position_embeddings
             }
-        };
-    let (max_position_embeddings, quantize): (usize, Option<Quantization>) =
-        get_max_positions_quantize().unwrap_or((4096, None));
+        } else {
+            max_default
+        }
+    } else {
+        max_default
+    };
+    let (prefix_caching, attention) = resolve_attention(&config, &args.lora_adapters);
+    tracing::info!("Using attention {attention} - Prefix caching {prefix_caching}");
+    std::env::set_var("USE_PREFIX_CACHING", prefix_caching);
+    std::env::set_var("ATTENTION", attention);
 
     let max_input_tokens = {
         match (args.max_input_tokens, args.max_input_length) {
diff --git a/load_tests/common.js b/load_tests/common.js
index e0a10595..d890bf67 100644
--- a/load_tests/common.js
+++ b/load_tests/common.js
@@ -33,13 +33,13 @@ export function get_options() {
             //     rate: 20,
             //     timeUnit: '1s',
             // },
-            load_test: {
-                executor: 'constant-arrival-rate',
-                duration: '60s',
-                preAllocatedVUs: 100,
-                rate: 1,
-                timeUnit: '1s',
-            },
+            // load_test: {
+            //     executor: 'constant-arrival-rate',
+            //     duration: '60s',
+            //     preAllocatedVUs: 100,
+            //     rate: 1,
+            //     timeUnit: '1s',
+            // },
             // breakpoint: {
             //     executor: 'ramping-arrival-rate', //Assure load increase if the system slows
             //     preAllocatedVUs: 300,
@@ -47,12 +47,12 @@ export function get_options() {
             //         { duration: '60s', target: 100 }, // just slowly ramp-up to a HUGE load
             //     ],
             // },
-            // throughput: {
-            //     executor: 'shared-iterations',
-            //     vus: 100,
-            //     iterations: 200,
-            //     maxDuration: '40s',
-            // },
+            throughput: {
+                executor: 'shared-iterations',
+                vus: 100,
+                iterations: 200,
+                maxDuration: '40s',
+            },
         },
     };
 }
diff --git a/proto/v3/generate.proto b/proto/v3/generate.proto
index 68eea7ac..34894bda 100644
--- a/proto/v3/generate.proto
+++ b/proto/v3/generate.proto
@@ -137,6 +137,8 @@ message Request {
   optional string adapter_id = 11;
   /// Prefix length that can be retrieved from the KV cache.
   uint32 prefix_len = 12;
+  /// Context truncation
+  bool add_special_tokens = 13;
 }
 
 message Batch {
diff --git a/router/src/infer/mod.rs b/router/src/infer/mod.rs
index 81c0d38f..240282d9 100644
--- a/router/src/infer/mod.rs
+++ b/router/src/infer/mod.rs
@@ -120,10 +120,11 @@ impl Infer {
     ) -> Result<Option<tokenizers::Encoding>, InferError> {
         // Tokenize request
         let inputs = request.inputs;
+        let add_special_tokens = request.add_special_tokens;
         let truncate = request.parameters.truncate;
         let encoding = self
             .validation
-            .tokenize(inputs, truncate)
+            .tokenize(inputs, add_special_tokens, truncate)
             .await
             .map_err(|err| {
                 tracing::error!("Tokenization {err}");
diff --git a/router/src/lib.rs b/router/src/lib.rs
index ce4f7c46..979f6dd1 100644
--- a/router/src/lib.rs
+++ b/router/src/lib.rs
@@ -22,6 +22,16 @@ pub enum Attention {
     FlashInfer,
 }
 
+impl Attention {
+    pub fn block_size(&self) -> u32 {
+        match self {
+            Attention::FlashDecoding => 256,
+            Attention::FlashInfer => 1,
+            Attention::Paged => 16,
+        }
+    }
+}
+
 #[derive(Debug)]
 pub struct ParseError;
 
@@ -1072,6 +1082,16 @@ pub(crate) struct GenerateRequest {
     pub inputs: String,
     #[serde(default = "default_parameters")]
     pub parameters: GenerateParameters,
+
+    /// This is used internally because some requests
+    /// already contain the templated input therefore
+    /// we shouldn't add the special tokens.
+    #[serde(default = "default_true", skip)]
+    pub add_special_tokens: bool,
+}
+
+fn default_true() -> bool {
+    true
 }
 
 #[derive(Clone, Debug, Deserialize, ToSchema)]
@@ -1089,6 +1109,7 @@ impl From<CompatGenerateRequest> for GenerateRequest {
     fn from(req: CompatGenerateRequest) -> Self {
         Self {
             inputs: req.inputs,
+            add_special_tokens: true,
             parameters: req.parameters,
         }
     }
diff --git a/router/src/server.rs b/router/src/server.rs
index 8ebd1a33..f273a786 100644
--- a/router/src/server.rs
+++ b/router/src/server.rs
@@ -158,6 +158,7 @@ async fn get_chat_tokenize(
 
     let generate_request = GenerateRequest {
         inputs,
+        add_special_tokens: false,
         parameters: GenerateParameters {
             best_of: None,
             temperature,
@@ -754,6 +755,7 @@ async fn completions(
         .iter()
         .map(|prompt| GenerateRequest {
             inputs: prompt.to_string(),
+            add_special_tokens: true,
             parameters: GenerateParameters {
                 best_of: None,
                 temperature,
@@ -1180,6 +1182,7 @@ async fn chat_completions(
     // build the request passing some parameters
     let generate_request = GenerateRequest {
         inputs: inputs.to_string(),
+        add_special_tokens: false,
         parameters: GenerateParameters {
             best_of: None,
             temperature,
@@ -1386,6 +1389,7 @@ async fn vertex_compatibility(
         .map(|instance| {
             let generate_request = GenerateRequest {
                 inputs: instance.inputs.clone(),
+                add_special_tokens: true,
                 parameters: GenerateParameters {
                     do_sample: true,
                     max_new_tokens: instance.parameters.as_ref().and_then(|p| p.max_new_tokens),
diff --git a/router/src/validation.rs b/router/src/validation.rs
index 0024723c..92491d88 100644
--- a/router/src/validation.rs
+++ b/router/src/validation.rs
@@ -95,6 +95,7 @@ impl Validation {
     pub async fn tokenize(
         &self,
         inputs: String,
+        add_special_tokens: bool,
         truncate: Option<usize>,
     ) -> Result<Option<(tokenizers::Encoding, Vec<Chunk>)>, ValidationError> {
         // If we have a fast tokenizer
@@ -104,7 +105,11 @@ impl Validation {
             // Send request to the background validation task
             // Unwrap is safe here
             sender
-                .send(((inputs, truncate), response_sender, Span::current()))
+                .send((
+                    (inputs, add_special_tokens, truncate),
+                    response_sender,
+                    Span::current(),
+                ))
                 .unwrap();
 
             // Await on response channel
@@ -121,11 +126,15 @@ impl Validation {
     async fn validate_input(
         &self,
         inputs: String,
+        add_special_tokens: bool,
         truncate: Option<usize>,
         max_new_tokens: Option<u32>,
     ) -> Result<(Vec<Chunk>, Option<Vec<u32>>, usize, u32), ValidationError> {
         // If we have a fast tokenizer
-        if let Some((encoding, inputs)) = self.tokenize(inputs.clone(), truncate).await? {
+        if let Some((encoding, inputs)) = self
+            .tokenize(inputs.clone(), add_special_tokens, truncate)
+            .await?
+        {
             // Create response channel
             let input_length = if let Some(truncate) = truncate {
                 std::cmp::min(encoding.len(), truncate)
@@ -158,7 +167,8 @@ impl Validation {
                 ));
             }
 
-            let input_ids = encoding.get_ids()[..input_length].to_owned();
+            let ids = encoding.get_ids();
+            let input_ids = ids[ids.len().saturating_sub(input_length)..].to_owned();
 
             metrics::histogram!("tgi_request_input_length").record(input_length as f64);
             Ok((inputs, Some(input_ids), input_length, max_new_tokens))
@@ -324,7 +334,12 @@ impl Validation {
 
         // Validate inputs
         let (inputs, input_ids, input_length, max_new_tokens) = self
-            .validate_input(request.inputs, truncate, max_new_tokens)
+            .validate_input(
+                request.inputs,
+                request.add_special_tokens,
+                truncate,
+                max_new_tokens,
+            )
             .await?;
 
         // TODO: we should build the FSM here and pass the compiled FSM instead of the grammar
@@ -401,6 +416,7 @@ impl Validation {
         Ok(ValidGenerateRequest {
             inputs,
             input_ids: input_ids.map(Arc::new),
+            add_special_tokens: request.add_special_tokens,
             decoder_input_details,
             input_length: input_length as u32,
             truncate: truncate.unwrap_or(self.max_input_length) as u32,
@@ -449,12 +465,15 @@ fn tokenizer_worker(
     mut receiver: mpsc::UnboundedReceiver<TokenizerRequest>,
 ) {
     // Loop over requests
-    while let Some(((inputs, truncate), response_tx, parent_span)) = receiver.blocking_recv() {
+    while let Some(((inputs, add_special_tokens, truncate), response_tx, parent_span)) =
+        receiver.blocking_recv()
+    {
         parent_span.in_scope(|| {
             response_tx
                 .send(prepare_input(
                     inputs,
                     truncate,
+                    add_special_tokens,
                     &tokenizer,
                     config.as_ref(),
                     preprocessor_config.as_ref(),
@@ -591,6 +610,7 @@ fn image_tokens_fixup(config: &Config, text: String) -> String {
 fn prepare_input(
     inputs: String,
     _truncate: Option<usize>,
+    add_special_tokens: bool,
     tokenizer: &Tokenizer,
     config: Option<&Config>,
     preprocessor_config: Option<&HubPreprocessorConfig>,
@@ -628,14 +648,14 @@ fn prepare_input(
 
     // Get the number of tokens in the input
     let encoding = tokenizer
-        .encode(tokenizer_query, true)
+        .encode(tokenizer_query, add_special_tokens)
         .map_err(|err| ValidationError::Tokenizer(err.to_string()))?;
 
     Ok((encoding, input_chunks))
 }
 
 type TokenizerRequest = (
-    (String, Option<usize>),
+    (String, bool, Option<usize>),
     oneshot::Sender<Result<(tokenizers::Encoding, Vec<Chunk>), ValidationError>>,
     Span,
 );
@@ -720,6 +740,7 @@ pub struct ValidGenerateRequest {
     pub input_ids: Option<Arc<Vec<u32>>>,
     pub input_length: u32,
     pub truncate: u32,
+    pub add_special_tokens: bool,
     pub decoder_input_details: bool,
     pub parameters: ValidParameters,
     pub stopping_parameters: ValidStoppingParameters,
@@ -826,7 +847,7 @@ mod tests {
 
         let max_new_tokens = 10;
         match validation
-            .validate_input("Hello".to_string(), None, Some(max_new_tokens))
+            .validate_input("Hello".to_string(), true, None, Some(max_new_tokens))
             .await
         {
             // Err(ValidationError::MaxNewTokens(1, 10)) => (),
@@ -861,7 +882,7 @@ mod tests {
 
         let max_new_tokens = 10;
         match validation
-            .validate_input("Hello".to_string(), None, Some(max_new_tokens))
+            .validate_input("Hello".to_string(), true, None, Some(max_new_tokens))
             .await
         {
             Err(ValidationError::MaxTotalTokens(6, 1, 10)) => (),
@@ -895,6 +916,7 @@ mod tests {
         match validation
             .validate(GenerateRequest {
                 inputs: "Hello".to_string(),
+                add_special_tokens: true,
                 parameters: GenerateParameters {
                     best_of: Some(2),
                     do_sample: false,
@@ -934,6 +956,7 @@ mod tests {
         match validation
             .validate(GenerateRequest {
                 inputs: "Hello".to_string(),
+                add_special_tokens: true,
                 parameters: GenerateParameters {
                     top_p: Some(1.0),
                     max_new_tokens: Some(5),
@@ -949,6 +972,7 @@ mod tests {
         match validation
             .validate(GenerateRequest {
                 inputs: "Hello".to_string(),
+                add_special_tokens: true,
                 parameters: GenerateParameters {
                     top_p: Some(0.99),
                     max_new_tokens: Some(5),
@@ -964,6 +988,7 @@ mod tests {
         let valid_request = validation
             .validate(GenerateRequest {
                 inputs: "Hello".to_string(),
+                add_special_tokens: true,
                 parameters: GenerateParameters {
                     top_p: None,
                     max_new_tokens: Some(5),
@@ -1002,6 +1027,7 @@ mod tests {
         match validation
             .validate(GenerateRequest {
                 inputs: "Hello".to_string(),
+                add_special_tokens: true,
                 parameters: GenerateParameters {
                     top_n_tokens: Some(5),
                     max_new_tokens: Some(5),
@@ -1017,6 +1043,7 @@ mod tests {
         validation
             .validate(GenerateRequest {
                 inputs: "Hello".to_string(),
+                add_special_tokens: true,
                 parameters: GenerateParameters {
                     top_n_tokens: Some(4),
                     max_new_tokens: Some(5),
@@ -1029,6 +1056,7 @@ mod tests {
         validation
             .validate(GenerateRequest {
                 inputs: "Hello".to_string(),
+                add_special_tokens: true,
                 parameters: GenerateParameters {
                     top_n_tokens: Some(0),
                     max_new_tokens: Some(5),
@@ -1041,6 +1069,7 @@ mod tests {
         let valid_request = validation
             .validate(GenerateRequest {
                 inputs: "Hello".to_string(),
+                add_special_tokens: true,
                 parameters: GenerateParameters {
                     top_n_tokens: None,
                     max_new_tokens: Some(5),
@@ -1089,6 +1118,7 @@ mod tests {
         let chunks = match validation
             .tokenize(
                 format!("test![](data:image/gif;base64,{})", PIXEL_GIF),
+                true,
                 None,
             )
             .await
@@ -1148,6 +1178,7 @@ mod tests {
                     "test![](data:image/gif;base64,{})![](data:image/gif;base64,{})",
                     PIXEL_GIF, PIXEL_GIF
                 ),
+                true,
                 None,
             )
             .await
diff --git a/rust-toolchain.toml b/rust-toolchain.toml
index 8c77896e..f392b161 100644
--- a/rust-toolchain.toml
+++ b/rust-toolchain.toml
@@ -1,5 +1,5 @@
 [toolchain]
 # Released on: June 13, 2024
 # https://releases.rs/docs/1.79.0/
-channel = "1.79.0"
+channel = "1.80.0"
 components = ["rustfmt", "clippy"]
diff --git a/server/Makefile b/server/Makefile
index 51ea8b32..9338b299 100644
--- a/server/Makefile
+++ b/server/Makefile
@@ -7,6 +7,7 @@ include Makefile-selective-scan
 include Makefile-lorax-punica
 include Makefile-fbgemm
 include Makefile-exllamav2
+include Makefile-flashinfer
 
 unit-tests:
 	pytest -s -vv -m "not private" tests
diff --git a/server/Makefile-flashinfer b/server/Makefile-flashinfer
new file mode 100644
index 00000000..3abb0491
--- /dev/null
+++ b/server/Makefile-flashinfer
@@ -0,0 +1,2 @@
+install-flashinfer:
+	pip install flashinfer==0.1.5 -i https://flashinfer.ai/whl/cu124/torch2.4
diff --git a/server/tests/conftest.py b/server/tests/conftest.py
index 16d2c408..d99771f8 100644
--- a/server/tests/conftest.py
+++ b/server/tests/conftest.py
@@ -1,7 +1,10 @@
 import pytest
-
+import os
 from text_generation_server.pb import generate_pb2
 
+os.environ["USE_PREFIX_CACHING"] = "1"
+os.environ["ATTENTION"] = "flashinfer"
+
 
 @pytest.fixture
 def default_pb_parameters():
diff --git a/server/text_generation_server/layers/attention/common.py b/server/text_generation_server/layers/attention/common.py
index f162230c..855f4dfc 100644
--- a/server/text_generation_server/layers/attention/common.py
+++ b/server/text_generation_server/layers/attention/common.py
@@ -9,26 +9,46 @@ if ATTENTION in {"flashinfer", "flashdecoding"}:
     @dataclass
     class Seqlen:
         input_lengths: torch.Tensor
+        prefix_lengths: torch.Tensor
         cu_seqlen_q: Optional[torch.Tensor]
         cu_seqlen_k: Optional[torch.Tensor]
+        max_q: int
+        max_k: int
 
-        def __init__(self, input_lengths):
+        def __init__(
+            self,
+            input_lengths,
+            prefix_lengths,
+            cu_seqlen_q=None,
+            max_q=None,
+            max_k=None,
+        ):
             self.input_lengths = input_lengths
+            self.prefix_lengths = prefix_lengths
             device = self.input_lengths.device
             shape = self.input_lengths.shape
-            cu_seqlen_q = torch.arange(
-                shape[0] + 1,
-                device=device,
-                dtype=torch.int32,
-            )
+            if cu_seqlen_q is None:
+                cu_seqlen_q = torch.arange(
+                    shape[0] + 1,
+                    device=device,
+                    dtype=torch.int32,
+                )
+                max_q = 1
+            else:
+                assert max_q is not None
+            assert max_k is not None
             cu_seqlen_k = torch.zeros(shape[-1] + 1, device=device, dtype=torch.int32)
+
             # cuda graphs don't like this and this is necessary to clamp within mistral
             # Although FA2 might not want the clamping
             # cu_seqlen_k[0] = 0
-            torch.cumsum(self.input_lengths, -1, out=cu_seqlen_k[1:])
+            total = self.input_lengths + self.prefix_lengths
+            torch.cumsum(total, -1, out=cu_seqlen_k[1:])
 
             self.cu_seqlen_q = cu_seqlen_q
             self.cu_seqlen_k = cu_seqlen_k
+            self.max_q = max_q
+            self.max_k = max_k
 
         def clamp(self, max):
             # Flash decoding doesn't need to clamp
@@ -39,6 +59,11 @@ else:
     @dataclass
     class Seqlen:
         input_lengths: torch.Tensor
+        prefix_lengths: torch.Tensor
+        cu_seqlen_q: torch.Tensor
+        max_q: int
+        max_k: int
 
         def clamp(self, max):
+            raise NotImplementedError("Not implemented seqlen for paged")
             return Seqlen(torch.clamp(self.input_lengths, max=max))
diff --git a/server/text_generation_server/layers/attention/cuda.py b/server/text_generation_server/layers/attention/cuda.py
index b3b7ea4f..4b588b5c 100644
--- a/server/text_generation_server/layers/attention/cuda.py
+++ b/server/text_generation_server/layers/attention/cuda.py
@@ -222,18 +222,15 @@ if ATTENTION == "flashinfer":
 
     def attention(
         q: torch.Tensor,
-        k: torch.Tensor,
-        v: torch.Tensor,
         key_cache: torch.Tensor,
         value_cache: torch.Tensor,
-        cu_seqlens,
-        max_s,
+        seqlen: Seqlen,
+        block_tables: torch.Tensor,
         softmax_scale,
         window_size_left=-1,
         causal=True,
         softcap=0.0,
     ):
-        assert window_size_left == -1, "Windowing is not supported with flash infer"
         from text_generation_server.layers.attention.flashinfer import (
             prefill_with_paged_kv_state,
         )
@@ -244,18 +241,17 @@ if ATTENTION == "flashinfer":
             paged_kv_cache=(key_cache, value_cache),
             logits_soft_cap=softcap,
             sm_scale=softmax_scale,
+            window_left=window_size_left,
         )
 
 elif V2:
 
     def attention(
         q,
-        k,
-        v,
         key_cache: torch.Tensor,
         value_cache: torch.Tensor,
-        cu_seqlens,
-        max_s,
+        seqlen: Seqlen,
+        block_tables: torch.Tensor,
         softmax_scale,
         window_size_left=-1,
         causal=True,
@@ -266,17 +262,17 @@ elif V2:
             raise ValueError("`window_size_left` must be > 0 or -1")
         return flash_attn_2_cuda.varlen_fwd(
             q,
-            k,
-            v,
+            key_cache,
+            value_cache,
             out,
-            cu_seqlens,
-            cu_seqlens,
+            seqlen.cu_seqlen_q,
+            seqlen.cu_seqlen_k,
             None,
             None,
+            block_tables,
             None,
-            None,
-            max_s,
-            max_s,
+            seqlen.max_q,
+            seqlen.max_k,
             0.0,
             softmax_scale,
             False,
diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py
index 4fa9e66d..e03cc30d 100644
--- a/server/text_generation_server/models/__init__.py
+++ b/server/text_generation_server/models/__init__.py
@@ -497,15 +497,14 @@ def get_model(
         else -1
     )
 
-    should_use_sliding_window = (
-        sliding_window is not None and sliding_window != -1 and SUPPORTS_WINDOWING
+    use_sliding_window = sliding_window is not None and sliding_window != -1
+    needs_sliding_window = (
+        max_input_tokens is not None and max_input_tokens > sliding_window
     )
-
-    if should_use_sliding_window:
-        if max_input_tokens is not None and max_input_tokens > sliding_window:
-            raise ValueError(
-                f"The backend {SYSTEM} does not support sliding window attention that is used by the model type {model_type}. To use this model nonetheless with the {SYSTEM} backend, please launch TGI with the argument `--max-input-tokens` smaller than sliding_window={sliding_window} (got here max_input_tokens={max_input_tokens})."
-            )
+    if use_sliding_window and needs_sliding_window and not SUPPORTS_WINDOWING:
+        raise ValueError(
+            f"The backend {SYSTEM} does not support sliding window attention that is used by the model type {model_type}. To use this model nonetheless with the {SYSTEM} backend, please launch TGI with the argument `--max-input-tokens` smaller than sliding_window={sliding_window} (got here max_input_tokens={max_input_tokens})."
+        )
 
     if model_type == DEEPSEEK_V2:
         if FLASH_ATTENTION:
diff --git a/server/text_generation_server/models/custom_modeling/flash_cohere_modeling.py b/server/text_generation_server/models/custom_modeling/flash_cohere_modeling.py
index 1eb8c6c3..fe19180a 100644
--- a/server/text_generation_server/models/custom_modeling/flash_cohere_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/flash_cohere_modeling.py
@@ -29,6 +29,7 @@ from text_generation_server.layers.attention import (
     paged_attention,
     attention,
     reshape_and_cache,
+    Seqlen,
 )
 from text_generation_server.utils.import_utils import SYSTEM
 from text_generation_server.layers import (
@@ -264,7 +265,7 @@ class FlashCohereAttention(torch.nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
     ):
         qkv = self.query_key_value(hidden_states)
@@ -296,12 +297,10 @@ class FlashCohereAttention(torch.nn.Module):
             # flash attention
             attn_output = attention(
                 query,
-                key,
-                value,
                 kv_cache[0],
                 kv_cache[1],
-                cu_seqlen_prefill,
-                max_s,
+                seqlen,
+                block_tables,
                 self.softmax_scale,
             )
         # Decode
@@ -313,7 +312,7 @@ class FlashCohereAttention(torch.nn.Module):
                 self.kv_head_mapping,
                 self.softmax_scale,
                 block_tables,
-                input_lengths,
+                seqlen,
                 max_s,
             )
 
@@ -388,7 +387,7 @@ class FlashCohereLayer(nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
     ):
         normed_hidden_states, res = self.input_layernorm(hidden_states, residual)
@@ -402,7 +401,7 @@ class FlashCohereLayer(nn.Module):
             kv_cache,
             block_tables,
             slots,
-            input_lengths,
+            seqlen,
             max_s,
         )
 
@@ -454,7 +453,7 @@ class FlashCohereModel(torch.nn.Module):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: torch.Tensor,
         max_s: int,
     ) -> torch.Tensor:
         hidden_states = self.embed_tokens(input_ids)
@@ -477,7 +476,7 @@ class FlashCohereModel(torch.nn.Module):
                 kv_cache[i],
                 block_tables,
                 slots,
-                input_lengths,
+                seqlen,
                 max_s,
             )
 
@@ -518,7 +517,7 @@ class FlashCohereForCausalLM(torch.nn.Module):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
         prefill_cache_indices: Optional[torch.Tensor],
         lm_head_indices: Optional[torch.Tensor] = None,
@@ -531,7 +530,7 @@ class FlashCohereForCausalLM(torch.nn.Module):
             kv_cache,
             block_tables,
             slots,
-            input_lengths,
+            seqlen,
             max_s,
         )
         if lm_head_indices is not None:
diff --git a/server/text_generation_server/models/custom_modeling/flash_dbrx_modeling.py b/server/text_generation_server/models/custom_modeling/flash_dbrx_modeling.py
index fc0dca5b..b82b5473 100644
--- a/server/text_generation_server/models/custom_modeling/flash_dbrx_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/flash_dbrx_modeling.py
@@ -29,6 +29,7 @@ from text_generation_server.layers.attention import (
     paged_attention,
     attention,
     reshape_and_cache,
+    Seqlen,
 )
 from text_generation_server.layers import (
     FastLinear,
@@ -309,7 +310,7 @@ class DbrxAttention(torch.nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
     ):
         qkv = self.query_key_value(hidden_states)
@@ -335,12 +336,10 @@ class DbrxAttention(torch.nn.Module):
             # flash attention
             attn_output = attention(
                 query,
-                torch.select(kv, dim=1, index=0),
-                torch.select(kv, dim=1, index=1),
                 kv_cache[0],
                 kv_cache[1],
-                cu_seqlen_prefill,
-                max_s,
+                seqlen,
+                block_tables,
                 self.softmax_scale,
             )
         # Decode
@@ -352,7 +351,7 @@ class DbrxAttention(torch.nn.Module):
                 self.kv_head_mapping,
                 self.softmax_scale,
                 block_tables,
-                input_lengths,
+                seqlen,
                 max_s,
             )
 
@@ -389,7 +388,7 @@ class DbrxNormAttentionNorm(nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
     ):
         normed_hidden_states, res = self.norm_1(hidden_states, residual)
@@ -403,7 +402,7 @@ class DbrxNormAttentionNorm(nn.Module):
             kv_cache,
             block_tables,
             slots,
-            input_lengths,
+            seqlen,
             max_s,
         )
 
@@ -622,7 +621,7 @@ class DbrxLayer(nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
     ):
         # Self Attention
@@ -635,7 +634,7 @@ class DbrxLayer(nn.Module):
             kv_cache,
             block_tables,
             slots,
-            input_lengths,
+            seqlen,
             max_s,
         )
 
@@ -679,7 +678,7 @@ class DbrxModel(torch.nn.Module):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
     ) -> torch.Tensor:
         hidden_states = self.embed_tokens(input_ids)
@@ -701,7 +700,7 @@ class DbrxModel(torch.nn.Module):
                 kv_cache[i],
                 block_tables,
                 slots,
-                input_lengths,
+                seqlen,
                 max_s,
             )
 
@@ -734,7 +733,7 @@ class FlashDbrxForCausalLM(torch.nn.Module):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
         prefill_cache_indices: Optional[torch.Tensor],
         lm_head_indices: Optional[torch.Tensor] = None,
@@ -747,7 +746,7 @@ class FlashDbrxForCausalLM(torch.nn.Module):
             kv_cache,
             block_tables,
             slots,
-            input_lengths,
+            seqlen,
             max_s,
         )
         if lm_head_indices is not None:
diff --git a/server/text_generation_server/models/custom_modeling/flash_deepseek_v2_modeling.py b/server/text_generation_server/models/custom_modeling/flash_deepseek_v2_modeling.py
index b25becd5..0585b40e 100644
--- a/server/text_generation_server/models/custom_modeling/flash_deepseek_v2_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/flash_deepseek_v2_modeling.py
@@ -29,8 +29,8 @@ from text_generation_server.layers.attention import (
     attention,
     paged_attention,
     reshape_and_cache,
+    Seqlen,
 )
-from text_generation_server.layers.attention.common import Seqlen
 from text_generation_server.layers.layernorm import FastRMSNorm
 from text_generation_server.layers.rotary import PositionRotaryEmbedding, get_mscale
 from text_generation_server.utils.import_utils import SYSTEM
@@ -298,7 +298,7 @@ class DeepseekV2Attention(torch.nn.Module):
         kv_cache: Tuple[torch.Tensor, torch.Tensor],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: Seqlen,
+        seqlen: Seqlen,
         max_s: int,
     ):
         if self.q_lora_rank is None:
@@ -363,12 +363,10 @@ class DeepseekV2Attention(torch.nn.Module):
             # flash attention
             attn_output = attention(
                 query,
-                key,
-                value,
                 kv_cache[0],
                 kv_cache[1],
-                cu_seqlen_prefill,
-                max_s,
+                seqlen,
+                block_tables,
                 self.softmax_scale,
             )
         # Decode
@@ -380,7 +378,7 @@ class DeepseekV2Attention(torch.nn.Module):
                 self.kv_head_mapping,
                 self.softmax_scale,
                 block_tables,
-                input_lengths,
+                seqlen,
                 max_s,
             )
 
@@ -666,7 +664,7 @@ class DeepseekV2Layer(nn.Module):
         kv_cache,
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: Seqlen,
+        seqlen: Seqlen,
         max_s: int,
     ):
         normed_hidden_states, residual = self.input_layernorm(hidden_states, residual)
@@ -680,7 +678,7 @@ class DeepseekV2Layer(nn.Module):
             kv_cache,
             block_tables,
             slots,
-            input_lengths,
+            seqlen,
             max_s,
         )
 
@@ -729,7 +727,7 @@ class DeepseekV2Model(torch.nn.Module):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
     ) -> torch.Tensor:
         hidden_states = self.embed_tokens(input_ids)
@@ -751,7 +749,7 @@ class DeepseekV2Model(torch.nn.Module):
                 kv_cache[i],
                 block_tables,
                 slots,
-                input_lengths,
+                seqlen,
                 max_s,
             )
 
@@ -781,7 +779,7 @@ class FlashDeepseekV2ForCausalLM(torch.nn.Module):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
         prefill_cache_indices: Optional[torch.Tensor],
         lm_head_indices: Optional[torch.Tensor] = None,
@@ -794,7 +792,7 @@ class FlashDeepseekV2ForCausalLM(torch.nn.Module):
             kv_cache,
             block_tables,
             slots,
-            input_lengths,
+            seqlen,
             max_s,
         )
         if lm_head_indices is not None:
diff --git a/server/text_generation_server/models/custom_modeling/flash_gemma2_modeling.py b/server/text_generation_server/models/custom_modeling/flash_gemma2_modeling.py
index faf0f325..d16e805f 100644
--- a/server/text_generation_server/models/custom_modeling/flash_gemma2_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/flash_gemma2_modeling.py
@@ -30,6 +30,7 @@ from text_generation_server.layers.attention import (
     paged_attention,
     attention,
     reshape_and_cache,
+    Seqlen,
 )
 from text_generation_server.layers import (
     TensorParallelRowLinear,
@@ -213,7 +214,7 @@ class FlashGemma2Attention(torch.nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
     ):
         qkv = self.query_key_value(hidden_states)
@@ -236,12 +237,10 @@ class FlashGemma2Attention(torch.nn.Module):
             # flash attention
             attn_output = attention(
                 query,
-                torch.select(kv, dim=1, index=0),
-                torch.select(kv, dim=1, index=1),
                 kv_cache[0],
                 kv_cache[1],
-                cu_seqlen_prefill,
-                max_s,
+                seqlen,
+                block_tables,
                 self.softmax_scale,
                 causal=self.causal,
                 window_size_left=self.window_size,
@@ -256,7 +255,7 @@ class FlashGemma2Attention(torch.nn.Module):
                 self.kv_head_mapping,
                 self.softmax_scale,
                 block_tables,
-                input_lengths,
+                seqlen,
                 max_s,
                 softcap=self.softcap,
             )
@@ -343,7 +342,7 @@ class FlashGemma2Layer(nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
     ):
         normed_hidden_states, res = self.input_layernorm(hidden_states, residual)
@@ -357,7 +356,7 @@ class FlashGemma2Layer(nn.Module):
             kv_cache,
             block_tables,
             slots,
-            input_lengths,
+            seqlen,
             max_s,
         )
 
@@ -408,7 +407,7 @@ class FlashGemma2Model(torch.nn.Module):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
     ) -> torch.Tensor:
         hidden_states = inputs_embeds
@@ -430,7 +429,7 @@ class FlashGemma2Model(torch.nn.Module):
                 kv_cache[i],
                 block_tables,
                 slots,
-                input_lengths,
+                seqlen,
                 max_s,
             )
 
@@ -477,7 +476,7 @@ class FlashGemma2ForCausalLM(torch.nn.Module):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
         prefill_cache_indices: Optional[torch.Tensor],
         lm_head_indices: Optional[torch.Tensor] = None,
@@ -491,7 +490,7 @@ class FlashGemma2ForCausalLM(torch.nn.Module):
             kv_cache,
             block_tables,
             slots,
-            input_lengths,
+            seqlen,
             max_s,
         )
         if lm_head_indices is not None:
diff --git a/server/text_generation_server/models/custom_modeling/flash_gemma_modeling.py b/server/text_generation_server/models/custom_modeling/flash_gemma_modeling.py
index 33738a59..34be4cb8 100644
--- a/server/text_generation_server/models/custom_modeling/flash_gemma_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/flash_gemma_modeling.py
@@ -30,6 +30,7 @@ from text_generation_server.layers.attention import (
     paged_attention,
     attention,
     reshape_and_cache,
+    Seqlen,
 )
 from text_generation_server.layers import (
     TensorParallelRowLinear,
@@ -207,7 +208,7 @@ class FlashGemmaAttention(torch.nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
     ):
         qkv = self.query_key_value(hidden_states)
@@ -230,12 +231,10 @@ class FlashGemmaAttention(torch.nn.Module):
             # flash attention
             attn_output = attention(
                 query,
-                torch.select(kv, dim=1, index=0),
-                torch.select(kv, dim=1, index=1),
                 kv_cache[0],
                 kv_cache[1],
-                cu_seqlen_prefill,
-                max_s,
+                seqlen,
+                block_tables,
                 self.softmax_scale,
                 causal=self.causal,
             )
@@ -248,7 +247,7 @@ class FlashGemmaAttention(torch.nn.Module):
                 self.kv_head_mapping,
                 self.softmax_scale,
                 block_tables,
-                input_lengths,
+                seqlen,
                 max_s,
             )
 
@@ -320,7 +319,7 @@ class FlashGemmaLayer(nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
     ):
         normed_hidden_states, res = self.input_layernorm(hidden_states, residual)
@@ -334,7 +333,7 @@ class FlashGemmaLayer(nn.Module):
             kv_cache,
             block_tables,
             slots,
-            input_lengths,
+            seqlen,
             max_s,
         )
 
@@ -382,7 +381,7 @@ class FlashGemmaModel(torch.nn.Module):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
     ) -> torch.Tensor:
         hidden_states = inputs_embeds
@@ -404,7 +403,7 @@ class FlashGemmaModel(torch.nn.Module):
                 kv_cache[i],
                 block_tables,
                 slots,
-                input_lengths,
+                seqlen,
                 max_s,
             )
 
@@ -449,7 +448,7 @@ class FlashGemmaForCausalLM(torch.nn.Module):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
         prefill_cache_indices: Optional[torch.Tensor],
         lm_head_indices: Optional[torch.Tensor] = None,
@@ -463,7 +462,7 @@ class FlashGemmaForCausalLM(torch.nn.Module):
             kv_cache,
             block_tables,
             slots,
-            input_lengths,
+            seqlen,
             max_s,
         )
         if lm_head_indices is not None:
diff --git a/server/text_generation_server/models/custom_modeling/flash_gpt2_modeling.py b/server/text_generation_server/models/custom_modeling/flash_gpt2_modeling.py
index d30b5a0a..403fa908 100644
--- a/server/text_generation_server/models/custom_modeling/flash_gpt2_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/flash_gpt2_modeling.py
@@ -29,6 +29,7 @@ from text_generation_server.layers.attention import (
     paged_attention,
     attention,
     reshape_and_cache,
+    Seqlen,
 )
 from text_generation_server.layers import (
     TensorParallelRowLinear,
@@ -213,7 +214,7 @@ class FlashGPT2Attention(torch.nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
     ):
         query, key, value = self.query_key_value(hidden_states).split(
@@ -230,12 +231,10 @@ class FlashGPT2Attention(torch.nn.Module):
             # flash attention
             attn_output = attention(
                 query,
-                key,
-                value,
                 kv_cache[0],
                 kv_cache[1],
-                cu_seqlen_prefill,
-                max_s,
+                seqlen,
+                block_tables,
                 self.softmax_scale,
             )
         # Decode
@@ -247,7 +246,7 @@ class FlashGPT2Attention(torch.nn.Module):
                 self.kv_head_mapping,
                 self.softmax_scale,
                 block_tables,
-                input_lengths,
+                seqlen,
                 max_s,
             )
 
@@ -316,7 +315,7 @@ class FlashGPT2Layer(nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
     ):
         residual = hidden_states
@@ -329,7 +328,7 @@ class FlashGPT2Layer(nn.Module):
             kv_cache,
             block_tables,
             slots,
-            input_lengths,
+            seqlen,
             max_s,
         )
 
@@ -382,7 +381,7 @@ class FlashGPT2Model(torch.nn.Module):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
         true_max_s: int,
         prefill_cache_indices: Optional[torch.Tensor],
@@ -398,7 +397,7 @@ class FlashGPT2Model(torch.nn.Module):
                 kv_cache[i],
                 block_tables,
                 slots,
-                input_lengths,
+                seqlen,
                 max_s,
             )
 
@@ -435,7 +434,7 @@ class FlashGPT2ForCausalLM(torch.nn.Module):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
         prefill_cache_indices: Optional[torch.Tensor] = None,
         lm_head_indices: Optional[torch.Tensor] = None,
@@ -451,7 +450,7 @@ class FlashGPT2ForCausalLM(torch.nn.Module):
             kv_cache,
             block_tables,
             slots,
-            input_lengths,
+            seqlen,
             max_s,
             true_max_s=max_s,
             prefill_cache_indices=prefill_cache_indices,
diff --git a/server/text_generation_server/models/custom_modeling/flash_gptj_modeling.py b/server/text_generation_server/models/custom_modeling/flash_gptj_modeling.py
index eb667384..35ab2791 100644
--- a/server/text_generation_server/models/custom_modeling/flash_gptj_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/flash_gptj_modeling.py
@@ -29,6 +29,7 @@ from text_generation_server.layers.attention import (
     paged_attention,
     attention,
     reshape_and_cache,
+    Seqlen,
 )
 from text_generation_server.layers import (
     TensorParallelRowLinear,
@@ -167,7 +168,7 @@ class FlashGPTJAttention(torch.nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
     ):
         query, key, value = self.query_key_value(hidden_states).split(
@@ -192,10 +193,10 @@ class FlashGPTJAttention(torch.nn.Module):
             # flash attention
             attn_output = attention(
                 query,
-                key,
-                value,
-                cu_seqlen_prefill,
-                max_s,
+                kv_cache[0],
+                kv_cache[1],
+                seqlen,
+                block_tables,
                 self.softmax_scale,
             )
         # Decode
@@ -207,7 +208,7 @@ class FlashGPTJAttention(torch.nn.Module):
                 self.kv_head_mapping,
                 self.softmax_scale,
                 block_tables,
-                input_lengths,
+                seqlen,
                 max_s,
             )
 
@@ -268,7 +269,7 @@ class FlashGPTJLayer(nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
     ):
         hidden_states, residual = self.input_layernorm(hidden_states, residual)
@@ -281,7 +282,7 @@ class FlashGPTJLayer(nn.Module):
             kv_cache,
             block_tables,
             slots,
-            input_lengths,
+            seqlen,
             max_s,
         )
 
@@ -328,7 +329,7 @@ class FlashGPTJModel(torch.nn.Module):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
         prefill_cache_indices: Optional[torch.Tensor],
     ) -> torch.Tensor:
@@ -351,7 +352,7 @@ class FlashGPTJModel(torch.nn.Module):
                 kv_cache[i],
                 block_tables,
                 slots,
-                input_lengths,
+                seqlen,
                 max_s,
             )
 
@@ -382,7 +383,7 @@ class FlashGPTJForCausalLM(torch.nn.Module):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
         prefill_cache_indices: Optional[torch.Tensor] = None,
         lm_head_indices: Optional[torch.Tensor] = None,
@@ -395,7 +396,7 @@ class FlashGPTJForCausalLM(torch.nn.Module):
             kv_cache,
             block_tables,
             slots,
-            input_lengths,
+            seqlen,
             max_s,
             prefill_cache_indices=prefill_cache_indices,
         )
diff --git a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py
index 3253d2dc..5b228f9f 100644
--- a/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/flash_llama_modeling.py
@@ -32,6 +32,7 @@ from text_generation_server.layers.attention import (
     paged_attention,
     attention,
     reshape_and_cache,
+    Seqlen,
 )
 from text_generation_server.layers import (
     TensorParallelRowLinear,
@@ -194,7 +195,7 @@ class FlashLlamaAttention(torch.nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
         adapter_data,
     ):
@@ -218,12 +219,10 @@ class FlashLlamaAttention(torch.nn.Module):
             # flash attention
             attn_output = attention(
                 query,
-                torch.select(kv, dim=1, index=0),
-                torch.select(kv, dim=1, index=1),
                 kv_cache[0],
                 kv_cache[1],
-                cu_seqlen_prefill,
-                max_s,
+                seqlen,
+                block_tables,
                 self.softmax_scale,
             )
         # Decode
@@ -235,7 +234,7 @@ class FlashLlamaAttention(torch.nn.Module):
                 self.kv_head_mapping,
                 self.softmax_scale,
                 block_tables,
-                input_lengths,
+                seqlen,
                 max_s,
             )
 
@@ -375,7 +374,7 @@ class FlashLlamaLayer(nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
         adapter_data,
     ):
@@ -390,7 +389,7 @@ class FlashLlamaLayer(nn.Module):
             kv_cache,
             block_tables,
             slots,
-            input_lengths,
+            seqlen,
             max_s,
             adapter_data,
         )
@@ -479,7 +478,7 @@ class FlashLlamaModel(torch.nn.Module):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
         true_max_s: int,
         prefill_cache_indices: Optional[torch.Tensor],
@@ -504,7 +503,7 @@ class FlashLlamaModel(torch.nn.Module):
                 kv_cache[i],
                 block_tables,
                 slots,
-                input_lengths,
+                seqlen,
                 max_s,
                 adapter_data,
             )
@@ -548,7 +547,7 @@ class FlashLlamaForCausalLM(torch.nn.Module):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
         prefill_cache_indices: Optional[torch.Tensor] = None,
         lm_head_indices: Optional[torch.Tensor] = None,
@@ -562,7 +561,7 @@ class FlashLlamaForCausalLM(torch.nn.Module):
             kv_cache,
             block_tables,
             slots,
-            input_lengths,
+            seqlen,
             max_s,
             true_max_s=max_s,
             prefill_cache_indices=prefill_cache_indices,
diff --git a/server/text_generation_server/models/custom_modeling/flash_mistral_modeling.py b/server/text_generation_server/models/custom_modeling/flash_mistral_modeling.py
index 5a150267..30ca3faf 100644
--- a/server/text_generation_server/models/custom_modeling/flash_mistral_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/flash_mistral_modeling.py
@@ -31,6 +31,7 @@ from text_generation_server.layers.attention import (
     paged_attention,
     attention,
     reshape_and_cache,
+    Seqlen,
 )
 from text_generation_server.layers import (
     TensorParallelRowLinear,
@@ -185,7 +186,7 @@ class MistralAttention(torch.nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
         prefill_cache_indices,
         adapter_data,
@@ -217,12 +218,10 @@ class MistralAttention(torch.nn.Module):
             # flash attention
             attn_output = attention(
                 query,
-                torch.select(kv, dim=1, index=0),
-                torch.select(kv, dim=1, index=1),
                 kv_cache[0],
                 kv_cache[1],
-                cu_seqlen_prefill,
-                max_s,
+                seqlen,
+                block_tables,
                 self.softmax_scale,
                 window_size_left=self.max_past,
             )
@@ -235,7 +234,7 @@ class MistralAttention(torch.nn.Module):
                 self.kv_head_mapping,
                 self.softmax_scale,
                 block_tables,
-                input_lengths,
+                seqlen,
                 max_s,
             )
 
@@ -356,7 +355,7 @@ class MistralLayer(nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
         prefill_cache_indices,
         adapter_data,
@@ -372,7 +371,7 @@ class MistralLayer(nn.Module):
             kv_cache,
             block_tables,
             slots,
-            input_lengths,
+            seqlen,
             max_s,
             prefill_cache_indices,
             adapter_data,
@@ -424,7 +423,7 @@ class MistralModel(torch.nn.Module):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
         true_max_s: int,
         prefill_cache_indices: Optional[torch.Tensor],
@@ -448,7 +447,7 @@ class MistralModel(torch.nn.Module):
                 kv_cache[i],
                 block_tables,
                 slots,
-                input_lengths,
+                seqlen,
                 max_s,
                 prefill_cache_indices,
                 adapter_data,
@@ -499,7 +498,7 @@ class FlashMistralForCausalLM(torch.nn.Module):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
         prefill_cache_indices: Optional[torch.Tensor],
         lm_head_indices: Optional[torch.Tensor] = None,
@@ -512,7 +511,7 @@ class FlashMistralForCausalLM(torch.nn.Module):
         elif self.max_past is not None:
             # Clamp in decode mode as paged attention requires clamped values whereas the flash attention
             # kernel requires the true values
-            input_lengths = input_lengths.clamp(max=self.max_past_tensor)
+            seqlen = seqlen.clamp(max=self.max_past_tensor)
 
         inputs_embeds = self.embed_tokens(input_ids)
         hidden_states = self.model(
@@ -522,7 +521,7 @@ class FlashMistralForCausalLM(torch.nn.Module):
             kv_cache,
             block_tables,
             slots,
-            input_lengths,
+            seqlen,
             max_s,
             true_max_s,
             prefill_cache_indices,
diff --git a/server/text_generation_server/models/custom_modeling/flash_mixtral_modeling.py b/server/text_generation_server/models/custom_modeling/flash_mixtral_modeling.py
index ad426ffe..c5d60af1 100644
--- a/server/text_generation_server/models/custom_modeling/flash_mixtral_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/flash_mixtral_modeling.py
@@ -35,6 +35,7 @@ from text_generation_server.layers.attention import (
     paged_attention,
     attention,
     reshape_and_cache,
+    Seqlen,
 )
 from text_generation_server.layers import (
     FastLinear,
@@ -243,7 +244,7 @@ class MixtralAttention(torch.nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
         prefill_cache_indices,
     ):
@@ -274,12 +275,10 @@ class MixtralAttention(torch.nn.Module):
             # flash attention
             attn_output = attention(
                 query,
-                torch.select(kv, dim=1, index=0),
-                torch.select(kv, dim=1, index=1),
                 kv_cache[0],
                 kv_cache[1],
-                cu_seqlen_prefill,
-                max_s,
+                seqlen,
+                block_tables,
                 self.softmax_scale,
                 window_size_left=self.max_past,
             )
@@ -292,7 +291,7 @@ class MixtralAttention(torch.nn.Module):
                 self.kv_head_mapping,
                 self.softmax_scale,
                 block_tables,
-                input_lengths,
+                seqlen,
                 max_s,
             )
 
@@ -498,7 +497,7 @@ class MixtralLayer(nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
         prefill_cache_indices,
     ):
@@ -513,7 +512,7 @@ class MixtralLayer(nn.Module):
             kv_cache,
             block_tables,
             slots,
-            input_lengths,
+            seqlen,
             max_s,
             prefill_cache_indices,
         )
@@ -568,7 +567,7 @@ class MixtralModel(torch.nn.Module):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
         true_max_s: int,
         prefill_cache_indices: Optional[torch.Tensor],
@@ -592,7 +591,7 @@ class MixtralModel(torch.nn.Module):
                 kv_cache[i],
                 block_tables,
                 slots,
-                input_lengths,
+                seqlen,
                 max_s,
                 prefill_cache_indices,
             )
@@ -627,7 +626,7 @@ class FlashMixtralForCausalLM(torch.nn.Module):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
         prefill_cache_indices: Optional[torch.Tensor],
         lm_head_indices: Optional[torch.Tensor] = None,
@@ -640,7 +639,7 @@ class FlashMixtralForCausalLM(torch.nn.Module):
         elif self.max_past is not None:
             # Clamp in decode mode as paged attention requires clamped values whereas the flash attention
             # kernel requires the true values
-            input_lengths = input_lengths.clamp(max=self.max_past_tensor)
+            seqlen = seqlen.clamp(max=self.max_past_tensor)
 
         hidden_states = self.model(
             input_ids,
@@ -649,7 +648,7 @@ class FlashMixtralForCausalLM(torch.nn.Module):
             kv_cache,
             block_tables,
             slots,
-            input_lengths,
+            seqlen,
             max_s,
             true_max_s,
             prefill_cache_indices,
diff --git a/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py b/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py
index b684e035..fda648f9 100644
--- a/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/flash_neox_modeling.py
@@ -31,6 +31,7 @@ from text_generation_server.layers.attention import (
     paged_attention,
     attention,
     reshape_and_cache,
+    Seqlen,
 )
 from text_generation_server.layers import (
     TensorParallelRowLinear,
@@ -147,7 +148,7 @@ class FlashNeoxAttention(torch.nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
     ):
         qkv = self.query_key_value(hidden_states)
@@ -171,12 +172,10 @@ class FlashNeoxAttention(torch.nn.Module):
             # flash attention
             attn_output = attention(
                 qkv[:, 0],
-                qkv[:, 1],
-                qkv[:, 2],
                 kv_cache[0],
                 kv_cache[1],
-                cu_seqlen_prefill,
-                max_s,
+                seqlen,
+                block_tables,
                 self.softmax_scale,
             )
         # Decode
@@ -188,7 +187,7 @@ class FlashNeoxAttention(torch.nn.Module):
                 self.kv_head_mapping,
                 self.softmax_scale,
                 block_tables,
-                input_lengths,
+                seqlen,
                 max_s,
             )
 
@@ -258,7 +257,7 @@ class FlashNeoXLayer(nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
     ):
         if self.use_parallel_residual:
@@ -272,7 +271,7 @@ class FlashNeoXLayer(nn.Module):
                 kv_cache,
                 block_tables,
                 slots,
-                input_lengths,
+                seqlen,
                 max_s,
             )
 
@@ -296,7 +295,7 @@ class FlashNeoXLayer(nn.Module):
                 kv_cache,
                 block_tables,
                 slots,
-                input_lengths,
+                seqlen,
                 max_s,
             )
 
@@ -350,7 +349,7 @@ class FlashGPTNeoXModel(FlashGPTNeoXPreTrainedModel):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
     ) -> torch.Tensor:
         hidden_states = self.embed_in(input_ids)
@@ -372,7 +371,7 @@ class FlashGPTNeoXModel(FlashGPTNeoXPreTrainedModel):
                 kv_cache[i],
                 block_tables,
                 slots,
-                input_lengths,
+                seqlen,
                 max_s,
             )
 
@@ -404,7 +403,7 @@ class FlashGPTNeoXForCausalLM(FlashGPTNeoXPreTrainedModel):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
         prefill_cache_indices: Optional[torch.Tensor],
         lm_head_indices: Optional[torch.Tensor] = None,
@@ -417,7 +416,7 @@ class FlashGPTNeoXForCausalLM(FlashGPTNeoXPreTrainedModel):
             kv_cache,
             block_tables,
             slots,
-            input_lengths,
+            seqlen,
             max_s,
         )
         if lm_head_indices is not None:
diff --git a/server/text_generation_server/models/custom_modeling/flash_pali_gemma_modeling.py b/server/text_generation_server/models/custom_modeling/flash_pali_gemma_modeling.py
index e08a2aad..d044b492 100644
--- a/server/text_generation_server/models/custom_modeling/flash_pali_gemma_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/flash_pali_gemma_modeling.py
@@ -19,6 +19,7 @@ from torch import nn
 from typing import Optional, List, Tuple
 
 from text_generation_server.layers.tensor_parallel import TensorParallelColumnLinear
+from text_generation_server.layers.attention import Seqlen
 from text_generation_server.models.custom_modeling.vlm import (
     load_text_model,
     load_vision_model,
@@ -70,7 +71,7 @@ class PaliGemmaForConditionalGeneration(nn.Module):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
         prefill_cache_indices: Optional[torch.Tensor] = None,
         lm_head_indices: Optional[torch.Tensor] = None,
@@ -107,7 +108,7 @@ class PaliGemmaForConditionalGeneration(nn.Module):
             kv_cache=kv_cache,
             block_tables=block_tables,
             slots=slots,
-            input_lengths=input_lengths,
+            seqlen=seqlen,
             max_s=max_s,
         )
 
diff --git a/server/text_generation_server/models/custom_modeling/flash_phi_modeling.py b/server/text_generation_server/models/custom_modeling/flash_phi_modeling.py
index efe27c13..37adb8be 100644
--- a/server/text_generation_server/models/custom_modeling/flash_phi_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/flash_phi_modeling.py
@@ -10,6 +10,7 @@ from text_generation_server.layers.attention import (
     paged_attention,
     attention,
     reshape_and_cache,
+    Seqlen,
 )
 from text_generation_server.layers import (
     TensorParallelRowLinear,
@@ -159,7 +160,7 @@ class FlashPhiAttention(torch.nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
     ):
         # Compute query, key, value and split
@@ -192,12 +193,10 @@ class FlashPhiAttention(torch.nn.Module):
         if cu_seqlen_prefill is not None:
             attn_output = attention(
                 query,
-                torch.select(kv, dim=1, index=0),
-                torch.select(kv, dim=1, index=1),
                 kv_cache[0],
                 kv_cache[1],
-                cu_seqlen_prefill,
-                max_s,
+                seqlen,
+                block_tables,
                 self.softmax_scale,
             )
         # Decode
@@ -209,7 +208,7 @@ class FlashPhiAttention(torch.nn.Module):
                 self.kv_head_mapping,
                 self.softmax_scale,
                 block_tables,
-                input_lengths,
+                seqlen,
                 max_s,
             )
 
@@ -276,7 +275,7 @@ class FlashPhiLayer(nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
     ):
         hidden_states, res = self.input_layernorm(hidden_states, residual)
@@ -289,7 +288,7 @@ class FlashPhiLayer(nn.Module):
             kv_cache,
             block_tables,
             slots,
-            input_lengths,
+            seqlen,
             max_s,
         )
 
@@ -341,7 +340,7 @@ class FlashPhiModel(torch.nn.Module):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
     ) -> torch.Tensor:
         hidden_states = self.embed_tokens(input_ids)
@@ -363,7 +362,7 @@ class FlashPhiModel(torch.nn.Module):
                 kv_cache[i],
                 block_tables,
                 slots,
-                input_lengths,
+                seqlen,
                 max_s,
             )
 
@@ -396,7 +395,7 @@ class FlashPhiForCausalLM(torch.nn.Module):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
         prefill_cache_indices: Optional[torch.Tensor],
         lm_head_indices: Optional[torch.Tensor] = None,
@@ -409,7 +408,7 @@ class FlashPhiForCausalLM(torch.nn.Module):
             kv_cache,
             block_tables,
             slots,
-            input_lengths,
+            seqlen,
             max_s,
         )
         if lm_head_indices is not None:
diff --git a/server/text_generation_server/models/custom_modeling/flash_qwen2_modeling.py b/server/text_generation_server/models/custom_modeling/flash_qwen2_modeling.py
index 879b8abd..5aac28a3 100644
--- a/server/text_generation_server/models/custom_modeling/flash_qwen2_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/flash_qwen2_modeling.py
@@ -9,6 +9,7 @@ from text_generation_server.layers.attention import (
     paged_attention,
     attention,
     reshape_and_cache,
+    Seqlen,
 )
 from text_generation_server.layers import (
     TensorParallelRowLinear,
@@ -104,7 +105,7 @@ class Qwen2Attention(torch.nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
         prefill_cache_indices,
     ):
@@ -135,12 +136,10 @@ class Qwen2Attention(torch.nn.Module):
             # flash attention
             attn_output = attention(
                 query,
-                torch.select(kv, dim=1, index=0),
-                torch.select(kv, dim=1, index=1),
                 kv_cache[0],
                 kv_cache[1],
-                cu_seqlen_prefill,
-                max_s,
+                seqlen,
+                block_tables,
                 self.softmax_scale,
                 window_size_left=self.max_past,
             )
@@ -153,7 +152,7 @@ class Qwen2Attention(torch.nn.Module):
                 self.kv_head_mapping,
                 self.softmax_scale,
                 block_tables,
-                input_lengths,
+                seqlen,
                 max_s,
             )
 
@@ -225,7 +224,7 @@ class Qwen2Layer(nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
         prefill_cache_indices,
     ):
@@ -240,7 +239,7 @@ class Qwen2Layer(nn.Module):
             kv_cache,
             block_tables,
             slots,
-            input_lengths,
+            seqlen,
             max_s,
             prefill_cache_indices,
         )
@@ -296,7 +295,7 @@ class Qwen2Model(torch.nn.Module):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
         true_max_s: int,
         prefill_cache_indices: Optional[torch.Tensor],
@@ -320,7 +319,7 @@ class Qwen2Model(torch.nn.Module):
                 kv_cache[i],
                 block_tables,
                 slots,
-                input_lengths,
+                seqlen,
                 max_s,
                 prefill_cache_indices,
             )
@@ -361,7 +360,7 @@ class Qwen2ForCausalLM(torch.nn.Module):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
         prefill_cache_indices: Optional[torch.Tensor] = None,
         lm_head_indices: Optional[torch.Tensor] = None,
@@ -374,7 +373,7 @@ class Qwen2ForCausalLM(torch.nn.Module):
         elif self.max_past is not None:
             # Clamp in decode mode as paged attention requires clamped values whereas the flash attention
             # kernel requires the true values
-            input_lengths = input_lengths.clamp(max=self.max_past_tensor)
+            seqlen = seqlen.clamp(max=self.max_past_tensor)
 
         hidden_states = self.model(
             input_ids,
@@ -383,7 +382,7 @@ class Qwen2ForCausalLM(torch.nn.Module):
             kv_cache,
             block_tables,
             slots,
-            input_lengths,
+            seqlen,
             max_s,
             true_max_s,
             prefill_cache_indices,
diff --git a/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py b/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py
index c72a9b90..1c55dd91 100644
--- a/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/flash_rw_modeling.py
@@ -19,6 +19,7 @@ from text_generation_server.layers.attention import (
     attention,
     paged_attention,
     reshape_and_cache,
+    Seqlen,
 )
 
 
@@ -181,7 +182,7 @@ class FlashRWAttention(torch.nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
     ):
         qkv = self.query_key_value(hidden_states)
@@ -206,12 +207,10 @@ class FlashRWAttention(torch.nn.Module):
             # flash attention
             attn_output = attention(
                 query,
-                torch.select(kv, dim=1, index=0),
-                torch.select(kv, dim=1, index=1),
                 kv_cache[0],
                 kv_cache[1],
-                cu_seqlen_prefill,
-                max_s,
+                seqlen,
+                block_tables,
                 self.softmax_scale,
             )
         # Decode
@@ -223,7 +222,7 @@ class FlashRWAttention(torch.nn.Module):
                 self.kv_head_mapping,
                 self.softmax_scale,
                 block_tables,
-                input_lengths,
+                seqlen,
                 max_s,
             )
 
@@ -296,7 +295,7 @@ class FlashRWLargeAttention(torch.nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
     ):
         qkv = self.query_key_value(hidden_states)
@@ -343,7 +342,7 @@ class FlashRWLargeAttention(torch.nn.Module):
                 self.kv_head_mapping,
                 self.softmax_scale,
                 block_tables,
-                input_lengths,
+                seqlen,
                 max_s,
             )
 
@@ -429,7 +428,7 @@ class FlashRWLayer(nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
     ):
         if self.parallel_attn:
@@ -443,7 +442,7 @@ class FlashRWLayer(nn.Module):
                 kv_cache,
                 block_tables,
                 slots,
-                input_lengths,
+                seqlen,
                 max_s,
             )
 
@@ -465,7 +464,7 @@ class FlashRWLayer(nn.Module):
                 kv_cache,
                 block_tables,
                 slots,
-                input_lengths,
+                seqlen,
                 max_s,
             )
 
@@ -552,7 +551,7 @@ class FlashRWLargeLayer(nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
     ):
         # Layer norm.
@@ -567,7 +566,7 @@ class FlashRWLargeLayer(nn.Module):
             kv_cache,
             block_tables,
             slots,
-            input_lengths,
+            seqlen,
             max_s,
         )
 
@@ -628,7 +627,7 @@ class FlashRWModel(FlashRWPreTrainedModel):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
     ) -> torch.Tensor:
         hidden_states = self.word_embeddings(input_ids)
@@ -650,7 +649,7 @@ class FlashRWModel(FlashRWPreTrainedModel):
                 kv_cache[i],
                 block_tables,
                 slots,
-                input_lengths,
+                seqlen,
                 max_s,
             )
 
@@ -680,7 +679,7 @@ class FlashRWForCausalLM(FlashRWPreTrainedModel):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
         prefill_cache_indices: Optional[torch.Tensor],
         lm_head_indices: Optional[torch.Tensor] = None,
@@ -693,7 +692,7 @@ class FlashRWForCausalLM(FlashRWPreTrainedModel):
             kv_cache,
             block_tables,
             slots,
-            input_lengths,
+            seqlen,
             max_s,
         )
         if lm_head_indices is not None:
diff --git a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py
index 109304be..19025c4c 100644
--- a/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/flash_santacoder_modeling.py
@@ -9,6 +9,7 @@ from text_generation_server.layers.attention import (
     paged_attention,
     attention,
     reshape_and_cache,
+    Seqlen,
 )
 from text_generation_server.layers import (
     TensorParallelRowLinear,
@@ -268,7 +269,7 @@ class FlashMQAttention(torch.nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
     ):
         qkv = self.c_attn(hidden_states)
@@ -291,12 +292,10 @@ class FlashMQAttention(torch.nn.Module):
             # flash attention
             attn_output = attention(
                 query,
-                torch.select(key_value, dim=1, index=0),
-                torch.select(key_value, dim=1, index=1),
                 kv_cache[0],
                 kv_cache[1],
-                cu_seqlen_prefill,
-                max_s,
+                seqlen,
+                block_tables,
                 self.softmax_scale,
             )
         # Decode
@@ -308,7 +307,7 @@ class FlashMQAttention(torch.nn.Module):
                 self.kv_head_mapping,
                 self.softmax_scale,
                 block_tables,
-                input_lengths,
+                seqlen,
                 max_s,
             )
 
@@ -373,7 +372,7 @@ class Block(nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
     ):
         hidden_states, residual = self.ln_1(hidden_states, residual)
@@ -383,7 +382,7 @@ class Block(nn.Module):
             kv_cache,
             block_tables,
             slots,
-            input_lengths,
+            seqlen,
             max_s,
         )
 
@@ -437,7 +436,7 @@ class FlashSantacoderModel(nn.Module):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
     ) -> torch.Tensor:
         hidden_states = self.wte(input_ids) + self.wpe(position_ids)
@@ -454,7 +453,7 @@ class FlashSantacoderModel(nn.Module):
                 kv_cache[i],
                 block_tables,
                 slots,
-                input_lengths,
+                seqlen,
                 max_s,
             )
 
@@ -486,7 +485,7 @@ class FlashSantacoderForCausalLM(nn.Module):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
         prefill_cache_indices: Optional[torch.Tensor],
         lm_head_indices: Optional[torch.Tensor] = None,
@@ -499,7 +498,7 @@ class FlashSantacoderForCausalLM(nn.Module):
             kv_cache,
             block_tables,
             slots,
-            input_lengths,
+            seqlen,
             max_s,
         )
         if lm_head_indices is not None:
diff --git a/server/text_generation_server/models/custom_modeling/flash_starcoder2_modeling.py b/server/text_generation_server/models/custom_modeling/flash_starcoder2_modeling.py
index 200d4ef0..2f9ecd0d 100644
--- a/server/text_generation_server/models/custom_modeling/flash_starcoder2_modeling.py
+++ b/server/text_generation_server/models/custom_modeling/flash_starcoder2_modeling.py
@@ -30,6 +30,7 @@ from text_generation_server.layers.attention import (
     paged_attention,
     attention,
     reshape_and_cache,
+    Seqlen,
 )
 from text_generation_server.layers import (
     TensorParallelRowLinear,
@@ -209,7 +210,7 @@ class Starcoder2Attention(torch.nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
         prefill_cache_indices,
     ):
@@ -240,12 +241,10 @@ class Starcoder2Attention(torch.nn.Module):
             # flash attention
             attn_output = attention(
                 query,
-                torch.select(kv, dim=1, index=0),
-                torch.select(kv, dim=1, index=1),
                 kv_cache[0],
                 kv_cache[1],
-                cu_seqlen_prefill,
-                max_s,
+                seqlen,
+                block_tables,
                 self.softmax_scale,
                 window_size_left=self.max_past,
             )
@@ -258,7 +257,7 @@ class Starcoder2Attention(torch.nn.Module):
                 self.kv_head_mapping,
                 self.softmax_scale,
                 block_tables,
-                input_lengths,
+                seqlen,
                 max_s,
             )
 
@@ -381,7 +380,7 @@ class Starcoder2Layer(nn.Module):
         kv_cache,
         block_tables,
         slots,
-        input_lengths,
+        seqlen,
         max_s,
         prefill_cache_indices,
     ):
@@ -396,7 +395,7 @@ class Starcoder2Layer(nn.Module):
             kv_cache,
             block_tables,
             slots,
-            input_lengths,
+            seqlen,
             max_s,
             prefill_cache_indices,
         )
@@ -449,7 +448,7 @@ class Starcoder2Model(torch.nn.Module):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
         true_max_s: int,
         prefill_cache_indices: Optional[torch.Tensor],
@@ -473,7 +472,7 @@ class Starcoder2Model(torch.nn.Module):
                 kv_cache[i],
                 block_tables,
                 slots,
-                input_lengths,
+                seqlen,
                 max_s,
                 prefill_cache_indices,
             )
@@ -521,7 +520,7 @@ class FlashStarcoder2ForCausalLM(torch.nn.Module):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
         prefill_cache_indices: Optional[torch.Tensor],
         lm_head_indices: Optional[torch.Tensor] = None,
@@ -534,7 +533,7 @@ class FlashStarcoder2ForCausalLM(torch.nn.Module):
         elif self.max_past is not None:
             # Clamp in decode mode as paged attention requires clamped values whereas the flash attention
             # kernel requires the true values
-            input_lengths = input_lengths.clamp(max=self.max_past_tensor)
+            seqlen = seqlen.clamp(max=self.max_past_tensor)
 
         hidden_states = self.model(
             input_ids,
@@ -543,7 +542,7 @@ class FlashStarcoder2ForCausalLM(torch.nn.Module):
             kv_cache,
             block_tables,
             slots,
-            input_lengths,
+            seqlen,
             max_s,
             true_max_s,
             prefill_cache_indices,
diff --git a/server/text_generation_server/models/custom_modeling/idefics2.py b/server/text_generation_server/models/custom_modeling/idefics2.py
index 7e4deaf8..a829c374 100644
--- a/server/text_generation_server/models/custom_modeling/idefics2.py
+++ b/server/text_generation_server/models/custom_modeling/idefics2.py
@@ -25,6 +25,7 @@ from transformers.activations import ACT2FN
 from text_generation_server.models.custom_modeling.vlm import (
     load_text_model,
 )
+from text_generation_server.layers.attention import Seqlen
 from transformers.modeling_attn_mask_utils import _prepare_4d_attention_mask
 
 from text_generation_server.layers import (
@@ -740,7 +741,7 @@ class Idefics2ForConditionalGeneration(nn.Module):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
         prefill_cache_indices: Optional[torch.Tensor],
         lm_head_indices: Optional[torch.Tensor] = None,
@@ -826,7 +827,7 @@ class Idefics2ForConditionalGeneration(nn.Module):
             kv_cache=kv_cache,
             block_tables=block_tables,
             slots=slots,
-            input_lengths=input_lengths,
+            seqlen=seqlen,
             max_s=max_s,
             true_max_s=max_s,
             prefill_cache_indices=None,
diff --git a/server/text_generation_server/models/custom_modeling/llava_next.py b/server/text_generation_server/models/custom_modeling/llava_next.py
index 29f5b9c7..32e9d334 100644
--- a/server/text_generation_server/models/custom_modeling/llava_next.py
+++ b/server/text_generation_server/models/custom_modeling/llava_next.py
@@ -23,6 +23,7 @@ from torch import nn
 from transformers.activations import ACT2FN
 from transformers.image_processing_utils import select_best_resolution
 
+from text_generation_server.layers.attention import Seqlen
 from text_generation_server.models.custom_modeling.vlm import (
     load_text_model,
     load_vision_model,
@@ -170,7 +171,7 @@ class LlavaNextForConditionalGeneration(nn.Module):
         kv_cache: List[Tuple[torch.Tensor, torch.Tensor]],
         block_tables: torch.Tensor,
         slots: torch.Tensor,
-        input_lengths: torch.Tensor,
+        seqlen: Seqlen,
         max_s: int,
         prefill_cache_indices: Optional[torch.Tensor],
         lm_head_indices: Optional[torch.Tensor] = None,
@@ -276,7 +277,7 @@ class LlavaNextForConditionalGeneration(nn.Module):
             kv_cache=kv_cache,
             block_tables=block_tables,
             slots=slots,
-            input_lengths=input_lengths,
+            seqlen=seqlen,
             max_s=max_s,
             true_max_s=max_s,
             prefill_cache_indices=None,
diff --git a/server/text_generation_server/models/flash_causal_lm.py b/server/text_generation_server/models/flash_causal_lm.py
index dd4203e0..9a60d06c 100644
--- a/server/text_generation_server/models/flash_causal_lm.py
+++ b/server/text_generation_server/models/flash_causal_lm.py
@@ -43,7 +43,7 @@ from text_generation_server.models.globals import (
     ATTENTION,
     BLOCK_SIZE,
     CUDA_GRAPHS,
-    PREFIX_CACHING,
+    TGI_WIGGLE_ROOM,
     get_adapter_to_index,
 )
 from text_generation_server.layers.attention import Seqlen
@@ -189,16 +189,21 @@ class FlashCausalLMBatch(Batch):
     def batch_tokenized_inputs(
         cls, requests: Iterable[generate_pb2.Request], tokenizer
     ):
-        batch_inputs = []
-        max_truncation = 0
+        max_length = 0
+        all_input_ids = []
+        batch_size = 0
         for r in requests:
-            batch_inputs.append(concat_text_chunks(r.input_chunks.chunks))
-            max_truncation = max(max_truncation, r.truncate)
-
-        batch_tokenized_inputs = tokenizer(
-            batch_inputs, truncation=True, max_length=max_truncation
-        )["input_ids"]
-        return batch_tokenized_inputs
+            batch_size += 1
+            inputs = concat_text_chunks(r.input_chunks.chunks)
+            input_ids = tokenizer(
+                inputs,
+                truncation=True,
+                max_length=r.truncate,
+                add_special_tokens=r.add_special_tokens,
+            )["input_ids"]
+            max_length = max(max_length, len(input_ids))
+            all_input_ids.append(input_ids)
+        return all_input_ids
 
     @classmethod
     def from_tokenized(
@@ -257,22 +262,15 @@ class FlashCausalLMBatch(Batch):
             # request id -> idx in list mapping
             requests_idx_mapping[r.id] = i
 
-            tokenized_input = tokenized_input[-r.truncate :]
-            if (
-                tokenized_input[0] == tokenizer.bos_token_id
-                and tokenized_input[1] == tokenizer.bos_token_id
-            ):
-                tokenized_input = tokenized_input[1:]
-
             orig_input_length = len(tokenized_input)
 
-            if PREFIX_CACHING:
-                prefix_len = r.prefix_len
-                if prefix_len == orig_input_length:
-                    assert prefix_len > 0
-                    prefix_len -= 1
-            else:
-                prefix_len = 0
+            prefix_len = r.prefix_len
+            assert (
+                prefix_len <= orig_input_length
+            ), f"Prefix {prefix_len} vs input {orig_input_length}"
+            if prefix_len == orig_input_length:
+                assert prefix_len > 0
+                prefix_len -= 1
 
             prefix_ids.append(tokenized_input[:prefix_len])
             tokenized_input = tokenized_input[prefix_len:]
@@ -998,7 +996,7 @@ class FlashCausalLM(Model):
             config.sliding_window = None
 
         self.num_layers = config.num_hidden_layers
-        self.num_heads = config.num_attention_heads
+        self.num_heads = config.num_attention_heads // self.process_group.size()
         # Validation is done in the model itself
         if num_kv_heads is None:
             num_kv_heads = getattr(config, "num_key_value_heads", None)
@@ -1160,8 +1158,15 @@ class FlashCausalLM(Model):
             "block_tables": block_tables,
             "slots": slots,
             "input_lengths": input_lengths_tensor,
+            "prefix_lengths": prefix_lengths_tensor,
         }
-        input_lengths_ = Seqlen(input_lengths=input_lengths_tensor)
+        seqlen = Seqlen(
+            input_lengths=input_lengths_tensor,
+            prefix_lengths=prefix_lengths_tensor,
+            cu_seqlen_q=None,
+            max_q=1,
+            max_k=max_s,
+        )
         graph = torch.cuda.CUDAGraph()
         self.cuda_graphs[bs]["graph"] = graph
 
@@ -1204,7 +1209,7 @@ class FlashCausalLM(Model):
                 kv_cache=self.kv_cache,
                 block_tables=block_tables,
                 slots=slots,
-                input_lengths=input_lengths_,
+                seqlen=seqlen,
                 max_s=max_s,
                 prefill_cache_indices=None,
                 lm_head_indices=None,
@@ -1213,7 +1218,13 @@ class FlashCausalLM(Model):
             torch.cuda.synchronize()
 
             with torch.cuda.graph(graph, pool=MEM_POOL):
-                input_lengths_tensor = Seqlen(input_lengths=input_lengths_tensor)
+                seqlen = Seqlen(
+                    input_lengths=input_lengths_tensor,
+                    prefix_lengths=prefix_lengths_tensor,
+                    cu_seqlen_q=None,
+                    max_q=1,
+                    max_k=max_s,
+                )
                 logits, speculative_logits = self.model.forward(
                     input_ids=input_ids,
                     position_ids=position_ids,
@@ -1221,7 +1232,7 @@ class FlashCausalLM(Model):
                     kv_cache=self.kv_cache,
                     block_tables=block_tables,
                     slots=slots,
-                    input_lengths=input_lengths_tensor,
+                    seqlen=seqlen,
                     max_s=max_s,
                     prefill_cache_indices=None,
                     lm_head_indices=None,
@@ -1268,7 +1279,7 @@ class FlashCausalLM(Model):
 
         num_blocks = (
             # Leave 5% for some wiggle room
-            int((free_memory * 0.95) // total_cache_size)
+            int((free_memory * TGI_WIGGLE_ROOM) // total_cache_size)
             # Add batch.num_blocks as we allocated it above, so it is included in the peak memory.
             + batch_num_blocks
         )
@@ -1360,18 +1371,26 @@ class FlashCausalLM(Model):
 
         # Dummy value, some models (starcoder2) don't accept `None`.
         input_lengths = torch.ones(seqlen, dtype=torch.int32, device=self.device)
-        input_lengths = Seqlen(input_lengths=input_lengths)
+        prefix_lens_tensor = torch.zeros(seqlen, dtype=torch.int32, device=self.device)
+        cu_seqlen_prefill = torch.tensor(
+            [0, seqlen], device=self.device, dtype=torch.int32
+        )
+        seqlen = Seqlen(
+            input_lengths=input_lengths,
+            prefix_lengths=prefix_lens_tensor,
+            cu_seqlen_q=cu_seqlen_prefill,
+            max_q=1,
+            max_k=seqlen,
+        )
 
         # We pass a `cu_seqlen_prefill` in order not to have to deal with paged attention cache allocation/deallocation.
         self.model.forward(
             input_ids=input_ids,
             position_ids=position_ids,
-            cu_seqlen_prefill=torch.tensor(
-                [0, seqlen], device=self.device, dtype=torch.int32
-            ),
+            cu_seqlen_prefill=cu_seqlen_prefill,
             kv_cache=self.kv_cache,
             block_tables=None,
-            input_lengths=input_lengths,
+            seqlen=seqlen,
             slots=slots,
             max_s=seqlen,
             lm_head_indices=None,
@@ -1451,8 +1470,7 @@ class FlashCausalLM(Model):
             cuda_graph = None
 
         if cu_seqlen_prefill is not None or cuda_graph is None:
-            input_lengths = input_lengths + prefix_lens_tensor
-            if PREFIX_CACHING:
+            if ATTENTION == "flashinfer":
                 block_tables = block_tables_to_ragged(
                     block_tables=block_tables,
                     input_lengths=batch.input_lengths,
@@ -1462,11 +1480,18 @@ class FlashCausalLM(Model):
                 block_tables=block_tables,
                 cu_seqlen_prefill=cu_seqlen_prefill,
                 input_lengths=batch.input_lengths,
-                input_lengths_tensor=input_lengths,
+                input_lengths_tensor=input_lengths + prefix_lens_tensor,
                 prefix_lens=batch.prefix_lens,
                 prefix_lens_tensor=prefix_lens_tensor,
             ):
-                input_lengths = Seqlen(input_lengths=input_lengths)
+                max_k = (input_lengths + prefix_lens_tensor).max().item()
+                seqlen = Seqlen(
+                    input_lengths=input_lengths,
+                    prefix_lengths=prefix_lens_tensor,
+                    cu_seqlen_q=cu_seqlen_prefill,
+                    max_q=max_s,
+                    max_k=max_k,
+                )
                 logits, speculative_logits = self.model.forward(
                     input_ids=input_ids,
                     position_ids=position_ids,
@@ -1474,7 +1499,7 @@ class FlashCausalLM(Model):
                     kv_cache=kv_cache,
                     block_tables=block_tables,
                     slots=slots,
-                    input_lengths=input_lengths,
+                    seqlen=seqlen,
                     max_s=max_s,
                     prefill_cache_indices=batch.prefill_cache_indices,
                     lm_head_indices=lm_head_indices,
diff --git a/server/text_generation_server/models/globals.py b/server/text_generation_server/models/globals.py
index d5133f5e..6c518c2c 100644
--- a/server/text_generation_server/models/globals.py
+++ b/server/text_generation_server/models/globals.py
@@ -5,19 +5,22 @@ from typing import Dict, Optional
 
 from text_generation_server.utils.log import log_master
 
-PREFIX_CACHING = os.getenv("USE_PREFIX_CACHING", "0").lower() in {"1", "true"}
+PREFIX_CACHING = os.getenv("USE_PREFIX_CACHING").lower() in {"1", "true"}
 log_master(logger.info, f"Using prefix caching = {PREFIX_CACHING}")
-ATTENTION = os.getenv("ATTENTION", "flashinfer" if PREFIX_CACHING else "paged")
+ATTENTION = os.getenv("ATTENTION")
 _expected = {"paged", "flashdecoding", "flashinfer"}
 assert (
     ATTENTION in _expected
 ), f"Attention is not valid {ATTENTION}, expected {_expected}"
 log_master(logger.info, f"Using Attention = {ATTENTION}")
 
-if PREFIX_CACHING and ATTENTION != "flashinfer":
+if PREFIX_CACHING and ATTENTION not in {"flashinfer", "flashdecoding"}:
     raise RuntimeError("Prefix caching is only supported with flashinfer")
 
 MEM_POOL = torch.cuda.graph_pool_handle() if torch.cuda.is_available() else None
+TGI_WIGGLE_ROOM = float(os.getenv("TGI_WIGGLE_ROOM", "0.95"))
+assert TGI_WIGGLE_ROOM > 0
+assert TGI_WIGGLE_ROOM < 1
 
 # This is overridden by the cli
 BLOCK_SIZE: int
diff --git a/server/text_generation_server/models/vlm_causal_lm.py b/server/text_generation_server/models/vlm_causal_lm.py
index 2ed1a119..d6cb36fa 100644
--- a/server/text_generation_server/models/vlm_causal_lm.py
+++ b/server/text_generation_server/models/vlm_causal_lm.py
@@ -372,7 +372,14 @@ class VlmCausalLM(FlashCausalLM):
                 prefix_lens=batch.prefix_lens,
                 prefix_lens_tensor=prefix_lens_tensor,
             ):
-                input_lengths = Seqlen(input_lengths=input_lengths)
+                max_k = (input_lengths + prefix_lens_tensor).max().item()
+                seqlen = Seqlen(
+                    input_lengths=input_lengths,
+                    prefix_lengths=prefix_lens_tensor,
+                    cu_seqlen_q=cu_seqlen_prefill,
+                    max_q=max_s,
+                    max_k=max_k,
+                )
                 logits, speculative_logits = self.model.forward(
                     input_ids=input_ids,
                     position_ids=position_ids,
@@ -380,7 +387,7 @@ class VlmCausalLM(FlashCausalLM):
                     kv_cache=kv_cache,
                     block_tables=block_tables,
                     slots=slots,
-                    input_lengths=input_lengths,
+                    seqlen=seqlen,
                     max_s=max_s,
                     prefill_cache_indices=batch.prefill_cache_indices,
                     lm_head_indices=lm_head_indices,

From d5202c46f7c42ea81f75dbc99f6eb1c5697c6b40 Mon Sep 17 00:00:00 2001
From: drbh <david.richard.holtz@gmail.com>
Date: Thu, 29 Aug 2024 10:32:38 -0400
Subject: [PATCH 70/72] feat: add /v1/models endpoint (#2433)

* feat: add /v1/models endpoint

* feat: add /v1/models endpoint

* fix: remove unused type import

* fix: revert route typo

* fix: update docs with new endpoint

* fix: add to redocly ignore and lint
---
 .redocly.lint-ignore.yaml |  1 +
 docs/openapi.json         | 60 +++++++++++++++++++++++++++++++++++++++
 router/src/lib.rs         | 28 ++++++++++++++++++
 router/src/server.rs      | 29 ++++++++++++++++++-
 4 files changed, 117 insertions(+), 1 deletion(-)

diff --git a/.redocly.lint-ignore.yaml b/.redocly.lint-ignore.yaml
index 382c9ab6..13b80497 100644
--- a/.redocly.lint-ignore.yaml
+++ b/.redocly.lint-ignore.yaml
@@ -77,3 +77,4 @@ docs/openapi.json:
     - '#/paths/~1tokenize/post'
     - '#/paths/~1v1~1chat~1completions/post'
     - '#/paths/~1v1~1completions/post'
+    - '#/paths/~1v1~1models/get'
diff --git a/docs/openapi.json b/docs/openapi.json
index fd64a3ab..691705f2 100644
--- a/docs/openapi.json
+++ b/docs/openapi.json
@@ -556,6 +556,37 @@
           }
         }
       }
+    },
+    "/v1/models": {
+      "get": {
+        "tags": [
+          "Text Generation Inference"
+        ],
+        "summary": "Get model info",
+        "operationId": "openai_get_model_info",
+        "responses": {
+          "200": {
+            "description": "Served model info",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/ModelInfo"
+                }
+              }
+            }
+          },
+          "404": {
+            "description": "Model not found",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/ErrorResponse"
+                }
+              }
+            }
+          }
+        }
+      }
     }
   },
   "components": {
@@ -1747,6 +1778,35 @@
           }
         ]
       },
+      "ModelInfo": {
+        "type": "object",
+        "required": [
+          "id",
+          "object",
+          "created",
+          "owned_by"
+        ],
+        "properties": {
+          "created": {
+            "type": "integer",
+            "format": "int64",
+            "example": 1686935002,
+            "minimum": 0
+          },
+          "id": {
+            "type": "string",
+            "example": "gpt2"
+          },
+          "object": {
+            "type": "string",
+            "example": "model"
+          },
+          "owned_by": {
+            "type": "string",
+            "example": "openai"
+          }
+        }
+      },
       "OutputMessage": {
         "oneOf": [
           {
diff --git a/router/src/lib.rs b/router/src/lib.rs
index 979f6dd1..a1e1dadf 100644
--- a/router/src/lib.rs
+++ b/router/src/lib.rs
@@ -1261,6 +1261,34 @@ pub(crate) struct ErrorResponse {
     pub error_type: String,
 }
 
+#[derive(Serialize, Deserialize, ToSchema)]
+pub(crate) struct ModelInfo {
+    #[schema(example = "gpt2")]
+    pub id: String,
+    #[schema(example = "model")]
+    pub object: String,
+    #[schema(example = 1686935002)]
+    pub created: u64,
+    #[schema(example = "openai")]
+    pub owned_by: String,
+}
+
+#[derive(Serialize, Deserialize, ToSchema)]
+pub(crate) struct ModelsInfo {
+    #[schema(example = "list")]
+    pub object: String,
+    pub data: Vec<ModelInfo>,
+}
+
+impl Default for ModelsInfo {
+    fn default() -> Self {
+        ModelsInfo {
+            object: "list".to_string(),
+            data: Vec::new(),
+        }
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
diff --git a/router/src/server.rs b/router/src/server.rs
index f273a786..d3d34215 100644
--- a/router/src/server.rs
+++ b/router/src/server.rs
@@ -24,6 +24,7 @@ use crate::{
     VertexResponse,
 };
 use crate::{FunctionDefinition, HubPreprocessorConfig, ToolCall, ToolChoice, ToolType};
+use crate::{ModelInfo, ModelsInfo};
 use async_stream::__private::AsyncStream;
 use axum::extract::Extension;
 use axum::http::{HeaderMap, HeaderValue, Method, StatusCode};
@@ -116,6 +117,29 @@ async fn get_model_info(info: Extension<Info>) -> Json<Info> {
     Json(info.0)
 }
 
+#[utoipa::path(
+get,
+tag = "Text Generation Inference",
+path = "/v1/models",
+responses(
+(status = 200, description = "Served model info", body = ModelInfo),
+(status = 404, description = "Model not found", body = ErrorResponse),
+)
+)]
+#[instrument(skip(info))]
+/// Get model info
+async fn openai_get_model_info(info: Extension<Info>) -> Json<ModelsInfo> {
+    Json(ModelsInfo {
+        data: vec![ModelInfo {
+            id: info.0.model_id.clone(),
+            object: "model".to_string(),
+            created: 0, // TODO: determine how to get this
+            owned_by: info.0.model_id.clone(),
+        }],
+        ..Default::default()
+    })
+}
+
 #[utoipa::path(
     post,
     tag = "Text Generation Inference",
@@ -1505,6 +1529,7 @@ chat_completions,
 completions,
 tokenize,
 metrics,
+openai_get_model_info,
 ),
 components(
 schemas(
@@ -1557,6 +1582,7 @@ ToolCall,
 Function,
 FunctionDefinition,
 ToolChoice,
+ModelInfo,
 )
 ),
 tags(
@@ -2250,7 +2276,8 @@ async fn start(
         .route("/info", get(get_model_info))
         .route("/health", get(health))
         .route("/ping", get(health))
-        .route("/metrics", get(metrics));
+        .route("/metrics", get(metrics))
+        .route("/v1/models", get(openai_get_model_info));
 
     // Conditional AWS Sagemaker route
     let aws_sagemaker_route = if messages_api_enabled {

From 9883f3b40e8e76cecc9807c438fc08c11e602b7a Mon Sep 17 00:00:00 2001
From: "Wang, Yi" <yi.a.wang@intel.com>
Date: Thu, 29 Aug 2024 23:42:02 +0800
Subject: [PATCH 71/72] update doc with intel cpu part (#2420)

* update doc with intel cpu part

Signed-off-by: Wang, Yi A <yi.a.wang@intel.com>

* Apply suggestions from code review

we do not use latest ever in documentation, it causes too many issues for users. Release number get update on every release.

---------

Signed-off-by: Wang, Yi A <yi.a.wang@intel.com>
Co-authored-by: Nicolas Patry <patry.nicolas@protonmail.com>
---
 docs/source/installation_intel.md | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/docs/source/installation_intel.md b/docs/source/installation_intel.md
index b3843490..3084a436 100644
--- a/docs/source/installation_intel.md
+++ b/docs/source/installation_intel.md
@@ -12,7 +12,24 @@ volume=$PWD/data # share a volume with the Docker container to avoid downloading
 docker run --rm --privileged --cap-add=sys_nice \
     --device=/dev/dri \
     --ipc=host --shm-size 1g --net host -v $volume:/data \
-    ghcr.io/huggingface/text-generation-inference:2.2.0-intel \
+    ghcr.io/huggingface/text-generation-inference:2.2.0-intel-xpu \
+    --model-id $model --cuda-graphs 0
+```
+
+# Using TGI with Intel CPUs
+
+Intel® Extension for PyTorch (IPEX) also provides further optimizations for Intel CPUs. The IPEX provides optimization operations such as flash attention, page attention, Add + LayerNorm, ROPE and more.
+
+On a server powered by Intel CPU, TGI can be launched with the following command:
+
+```bash
+model=teknium/OpenHermes-2.5-Mistral-7B
+volume=$PWD/data # share a volume with the Docker container to avoid downloading weights every run
+
+docker run --rm --privileged --cap-add=sys_nice \
+    --device=/dev/dri \
+    --ipc=host --shm-size 1g --net host -v $volume:/data \
+    ghcr.io/huggingface/text-generation-inference:2.2.0-intel-cpu \
     --model-id $model --cuda-graphs 0
 ```
 

From d9fbbaafb046bb423e31edaf9ccf8eecc2d5c33d Mon Sep 17 00:00:00 2001
From: Nicolas Patry <patry.nicolas@protonmail.com>
Date: Thu, 29 Aug 2024 17:44:54 +0200
Subject: [PATCH 72/72] Tied embeddings in MLP speculator. (#2473)

* Tied embeddings in MLP speculator.

* Fixing the scale_weight when users decide to not use the speculation as
much as defined in the config.

* Adding scaling support + optimize some ops.
---
 server/text_generation_server/layers/mlp.py   | 120 +++++++++++++++++-
 .../text_generation_server/models/__init__.py |   5 +
 2 files changed, 118 insertions(+), 7 deletions(-)

diff --git a/server/text_generation_server/layers/mlp.py b/server/text_generation_server/layers/mlp.py
index f08cb673..d33b41f3 100644
--- a/server/text_generation_server/layers/mlp.py
+++ b/server/text_generation_server/layers/mlp.py
@@ -45,12 +45,107 @@ class MLPSpeculatorLayerNorm(nn.Module):
         return x
 
 
+INV_SQRT2 = 2**-0.5
+
+
+def simple_norm(x: torch.Tensor, eps=1e-06):
+    xf = x
+    xf = xf * torch.rsqrt(xf.pow(2).mean(-1, keepdim=True) + eps)
+    x = xf.type_as(x)
+    return x * INV_SQRT2
+
+
+class MLPSpeculatorModelTied(torch.nn.Module):
+    def __init__(self, config, prefix, weights):
+        super().__init__()
+        self.config = config
+        self.n_predict = get_speculate()
+        self.hidden_size = config.hidden_size
+
+        self.emb = TensorParallelEmbedding(f"{prefix}.emb.0", weights)
+        self.proj0 = FastLinear.load(
+            config,
+            prefix=f"{prefix}.proj.0",
+            weights=weights,
+            bias=False,
+        )
+        self.proj1 = FastLinear.load(
+            config,
+            prefix=f"{prefix}.proj.1",
+            weights=weights,
+            bias=False,
+        )
+        self.head = FastLinear.load(config, f"{prefix}.head.0", weights, bias=False)
+        self.ln = MLPSpeculatorLayerNorm(
+            prefix=f"{prefix}.ln.0",
+            config=config,
+            weights=weights,
+        )
+
+        # Weights ensure that state_0 accounts for 50% of state magnitude by final head in expectation
+        self.state_weight = 0.5 ** (0.5 / self.n_predict) if self.n_predict > 0 else 1
+        self.activation = nn.GELU()
+        self.vsize = config.vocab_size
+        self.inner_dim = config.speculator_config["inner_dim"]
+        self.top_k_tokens_per_head = [1] * self.n_predict
+        self.emb_weight = math.sqrt(1 - self.state_weight**2) * math.sqrt(
+            self.inner_dim / 2
+        )
+        self.emb.weight *= self.emb_weight
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        input_ids: torch.Tensor,
+    ):
+        top_k_tokens_per_head = self.top_k_tokens_per_head
+
+        # k indicates # of candidates
+        # h indicates # of generated tokens
+        state = hidden_states
+        b = state.size(0)
+        ind = input_ids.unsqueeze(0)
+        all_probs = torch.empty(
+            b, self.n_predict, self.vsize, device=state.device
+        )  # b k h v
+        assert (
+            len(top_k_tokens_per_head) == self.n_predict
+        ), f"You must provide a topk number for each head ({self.n_predict} heads, {len(top_k_tokens_per_head)} provided)"
+        for i in range(self.n_predict):
+            # Project and predict
+            z = self.emb(ind)
+            # z = z.mul(self.emb_weight)  # b k d
+            if i == 0:
+                state = self.proj0(state) * self.state_weight + z
+            else:
+                state = self.proj1(state) * self.state_weight + z
+            state = self.activation(self.ln(state))  # b k d
+            probs = F.log_softmax(self.head(state), dim=-1)  # b k v
+            _probs, preds = probs.topk(top_k_tokens_per_head[i], dim=-1)  # b k k'
+
+            # Update candidate set with new predictions
+
+            # Update distribution set with new logits
+            all_probs[:, i] = probs.exp()
+
+            # Update state, log_probs and ind for new predictions
+            state = state.unsqueeze(2).expand(
+                -1, -1, top_k_tokens_per_head[i], -1
+            )  # b k k' d
+            state = state.reshape(-1, b, state.size(3))  # b kk' d
+            ind = preds.view(-1, b)  # b kk'
+
+        speculative_logits = all_probs
+        return speculative_logits
+
+
 class MLPSpeculatorModel(torch.nn.Module):
     def __init__(self, config, prefix, weights):
         super().__init__()
         self.config = config
         self.n_predict = get_speculate()
         self.hidden_size = config.hidden_size
+
         self.emb = nn.ModuleList(
             [
                 TensorParallelEmbedding(f"{prefix}.emb.{i}", weights)
@@ -84,13 +179,15 @@ class MLPSpeculatorModel(torch.nn.Module):
         )
 
         # Weights ensure that state_0 accounts for 50% of state magnitude by final head in expectation
-        self.state_weight = 0.5 ** (0.5 / self.n_predict)
-        self.emb_weight = math.sqrt(1 - self.state_weight**2)
+        self.state_weight = 0.5 ** (0.5 / self.n_predict) if self.n_predict > 0 else 1
         self.activation = nn.GELU()
-        # TODO
         self.vsize = config.vocab_size
         self.inner_dim = config.speculator_config["inner_dim"]
         self.top_k_tokens_per_head = [1] * self.n_predict
+        self.emb_weight = math.sqrt(1 - self.state_weight**2) * math.sqrt(
+            self.inner_dim / 2
+        )
+        self.emb.weight *= self.emb_weight
 
     def forward(
         self,
@@ -113,7 +210,7 @@ class MLPSpeculatorModel(torch.nn.Module):
         for i in range(self.n_predict):
             # Project and predict
             z = self.emb[i](ind)
-            z = z.mul(self.emb_weight * math.sqrt(self.inner_dim / 2))  # b k d
+            # z = z.mul(self.emb_weight)  # b k d
             state = self.proj[i](state) * self.state_weight + z
             state = self.activation(self.ln[i](state))  # b k d
             probs = F.log_softmax(self.head[i](state), dim=-1)  # b k v
@@ -136,10 +233,11 @@ class MLPSpeculatorModel(torch.nn.Module):
 
 
 class MLPSpeculatorHead(nn.Module):
-    def __init__(self, lm_head, mlp_speculator):
+    def __init__(self, lm_head, mlp_speculator, scale_input: bool):
         super().__init__()
         self.lm_head = lm_head
         self.mlp_speculator = mlp_speculator
+        self.scale_input = scale_input
 
     def forward(
         self, input: torch.Tensor
@@ -150,6 +248,8 @@ class MLPSpeculatorHead(nn.Module):
             return logits, None
 
         input_ids = logits.argmax(dim=-1)
+        if self.scale_input:
+            input = simple_norm(input)
         speculative_logits = self.mlp_speculator(input, input_ids)
         return logits, speculative_logits
 
@@ -171,6 +271,12 @@ class MLPSpeculatorHead(nn.Module):
                         )
                     routing[k] = filename
 
-        mlp_speculator = MLPSpeculatorModel(config, "speculator", weights)
+        tie_weights = config.speculator_config.get("tie_weights", False)
+        if tie_weights:
+            mlp_speculator = MLPSpeculatorModelTied(config, "speculator", weights)
+        else:
+            mlp_speculator = MLPSpeculatorModel(config, "speculator", weights)
+        # This is used in https://huggingface.co/ibm-fms/llama3-70b-accelerator
+        scale_input = config.speculator_config.get("scale_input", False)
         lm_head = TensorParallelHead.load(config, prefix, weights)
-        return MLPSpeculatorHead(lm_head, mlp_speculator)
+        return MLPSpeculatorHead(lm_head, mlp_speculator, scale_input)
diff --git a/server/text_generation_server/models/__init__.py b/server/text_generation_server/models/__init__.py
index e03cc30d..52f332c1 100644
--- a/server/text_generation_server/models/__init__.py
+++ b/server/text_generation_server/models/__init__.py
@@ -458,6 +458,11 @@ def get_model(
                     revision=mlp_revision,
                     filename=filename,
                 )
+            speculator_dir_path = Path(mlp_speculator_config).parent
+            # if these are downloaded, they get converted to safetensors
+            filenames.extend(
+                [p for p in os.listdir(speculator_dir_path) if p.endswith(extension)]
+            )
             speculator = {
                 "path": Path(mlp_speculator_config).parent,
                 "model_paths": filenames,