From 91a705e1a971b0e8dc7649ce68a0748654a359e0 Mon Sep 17 00:00:00 2001 From: drbh Date: Mon, 29 Apr 2024 16:07:53 +0000 Subject: [PATCH] feat: prefer custom deserializer for complex message content --- router/src/infer.rs | 1436 +++++++++++++++++++++---------------------- router/src/lib.rs | 50 +- 2 files changed, 749 insertions(+), 737 deletions(-) diff --git a/router/src/infer.rs b/router/src/infer.rs index 139d3531..da0db072 100644 --- a/router/src/infer.rs +++ b/router/src/infer.rs @@ -4,7 +4,6 @@ use crate::{ ChatTemplateInputs, ChatTemplateVersions, Entry, GenerateRequest, GenerateStreamResponse, HubTokenizerConfig, Message, PrefillToken, Queue, Token, }; -use crate::{Content, SerializedMessage}; use crate::{FunctionRef, FunctionsMap, GrammarType, Properties, Tool, ToolType, Tools}; use futures::future::try_join_all; use minijinja::{Environment, ErrorKind, Template}; @@ -194,7 +193,6 @@ impl Infer { messages: Vec, grammar_with_prompt: Option<(GrammarType, String)>, ) -> Result { - println!("messages: {:?}", messages); self.chat_template .as_ref() .ok_or_else(|| InferError::TemplateError(ErrorKind::TemplateNotFound.into()))? @@ -361,50 +359,22 @@ impl ChatTemplate { mut messages: Vec, grammar_with_prompt: Option<(GrammarType, String)>, ) -> Result { - println!("--- messages: {:?}", messages); if self.use_default_tool_template { if let Some(last_message) = messages.last_mut() { - println!("\nlast_message: {:#?}\n", last_message); - if let Some((GrammarType::Json(tools), _tool_prompt)) = grammar_with_prompt { - println!("tools: {:?}", tools); - - // last_message.content = Some(format!( - // "{}\n---\n{}\n{}", - // last_message.content.as_deref().unwrap_or_default(), - // tool_prompt, - // tools - // )); - - last_message.content = Some(vec![Content { - r#type: "tool".to_string(), - text: None, - image_url: None, - }]) + if let Some((GrammarType::Json(tools), tool_prompt)) = grammar_with_prompt { + last_message.content = Some(format!( + "{}\n---\n{}\n{}", + last_message.content.as_deref().unwrap_or_default(), + tool_prompt, + tools + )); } } } self.template .render(ChatTemplateInputs { - messages: messages - .into_iter() - .map(|m| { - // - SerializedMessage { - role: m.role, - content: match m.content { - Some(content) => { - // - println!("content: {:?}", content); - "text".to_string() - } - None => "".to_string(), - }, - name: m.name, - tool_calls: m.tool_calls, - } - }) - .collect(), + messages, bos_token: self.bos_token.as_deref(), eos_token: self.eos_token.as_deref(), add_generation_prompt: true, @@ -965,732 +935,732 @@ impl InferError { } } -// // tests -// #[cfg(test)] -// mod tests { -// use crate::infer::raise_exception; -// use crate::ChatTemplateInputs; -// use crate::Message; -// use minijinja::Environment; +// tests +#[cfg(test)] +mod tests { + use crate::infer::raise_exception; + use crate::ChatTemplateInputs; + use crate::Message; + use minijinja::Environment; -// #[test] -// fn test_chat_template() { -// let env = Environment::new(); + #[test] + fn test_chat_template() { + let env = Environment::new(); -// let source = r#" -// {% for message in messages %} -// {% if message['role'] == 'system' %} -// {% if message['content']%} -// {{'### System:\n' + message['content']+'\n\n'}} -// {% endif %} -// {% elif message['role'] == 'user' %} -// {{'### User:\n' + message['content']+'\n\n'}} -// {% elif message['role'] == 'assistant' %} -// {{'### Assistant:\n' + message['content']}} -// {% endif %} -// {% if loop.last and add_generation_prompt %} -// {{ '### Assistant:\n' }} -// {% endif %} -// {% endfor %}"#; + let source = r#" + {% for message in messages %} + {% if message['role'] == 'system' %} + {% if message['content']%} + {{'### System:\n' + message['content']+'\n\n'}} + {% endif %} + {% elif message['role'] == 'user' %} + {{'### User:\n' + message['content']+'\n\n'}} + {% elif message['role'] == 'assistant' %} + {{'### Assistant:\n' + message['content']}} + {% endif %} + {% if loop.last and add_generation_prompt %} + {{ '### Assistant:\n' }} + {% endif %} + {% endfor %}"#; -// // trim all the whitespace -// let source = source -// .lines() -// .map(|line| line.trim()) -// .collect::>() -// .join(""); + // trim all the whitespace + let source = source + .lines() + .map(|line| line.trim()) + .collect::>() + .join(""); -// let tmpl = env.template_from_str(&source); + let tmpl = env.template_from_str(&source); -// let chat_template_inputs = ChatTemplateInputs { -// messages: vec![ -// Message { -// role: "user".to_string(), -// content: Some("Hi!".to_string()), -// name: None, -// tool_calls: None, -// }, -// Message { -// role: "assistant".to_string(), -// content: Some("Hello how can I help?".to_string()), -// name: None, -// tool_calls: None, -// }, -// Message { -// role: "user".to_string(), -// content: Some("What is Deep Learning?".to_string()), -// name: None, -// tool_calls: None, -// }, -// Message { -// role: "assistant".to_string(), -// content: Some("magic!".to_string()), -// name: None, -// tool_calls: None, -// }, -// ], -// bos_token: Some("[BOS]"), -// eos_token: Some("[EOS]"), -// add_generation_prompt: true, -// ..Default::default() -// }; + let chat_template_inputs = ChatTemplateInputs { + messages: vec![ + Message { + role: "user".to_string(), + content: Some("Hi!".to_string()), + name: None, + tool_calls: None, + }, + Message { + role: "assistant".to_string(), + content: Some("Hello how can I help?".to_string()), + name: None, + tool_calls: None, + }, + Message { + role: "user".to_string(), + content: Some("What is Deep Learning?".to_string()), + name: None, + tool_calls: None, + }, + Message { + role: "assistant".to_string(), + content: Some("magic!".to_string()), + name: None, + tool_calls: None, + }, + ], + bos_token: Some("[BOS]"), + eos_token: Some("[EOS]"), + add_generation_prompt: true, + ..Default::default() + }; -// let result = tmpl.unwrap().render(chat_template_inputs).unwrap(); + let result = tmpl.unwrap().render(chat_template_inputs).unwrap(); -// assert_eq!( -// result, -// "### User:\nHi!\n\n### Assistant:\nHello how can I help?### User:\nWhat is Deep Learning?\n\n### Assistant:\nmagic!### Assistant:\n" -// ); -// } + assert_eq!( + result, + "### User:\nHi!\n\n### Assistant:\nHello how can I help?### User:\nWhat is Deep Learning?\n\n### Assistant:\nmagic!### Assistant:\n" + ); + } -// #[test] -// fn test_chat_template_invalid_with_raise() { -// let mut env = Environment::new(); -// env.add_function("raise_exception", raise_exception); + #[test] + fn test_chat_template_invalid_with_raise() { + let mut env = Environment::new(); + env.add_function("raise_exception", raise_exception); -// let source = r#" -// {{ bos_token }} -// {% for message in messages %} -// {% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %} -// {{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }} -// {% endif %} -// {% if message['role'] == 'user' %} -// {{ '[INST] ' + message['content'] + ' [/INST]' }} -// {% elif message['role'] == 'assistant' %} -// {{ message['content'] + eos_token}} -// {% else %} -// {{ raise_exception('Only user and assistant roles are supported!') }} -// {% endif %} -// {% endfor %}"#; + let source = r#" + {{ bos_token }} + {% for message in messages %} + {% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %} + {{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }} + {% endif %} + {% if message['role'] == 'user' %} + {{ '[INST] ' + message['content'] + ' [/INST]' }} + {% elif message['role'] == 'assistant' %} + {{ message['content'] + eos_token}} + {% else %} + {{ raise_exception('Only user and assistant roles are supported!') }} + {% endif %} + {% endfor %}"#; -// // trim all the whitespace -// let source = source -// .lines() -// .map(|line| line.trim()) -// .collect::>() -// .join(""); + // trim all the whitespace + let source = source + .lines() + .map(|line| line.trim()) + .collect::>() + .join(""); -// let tmpl = env.template_from_str(&source); + let tmpl = env.template_from_str(&source); -// let chat_template_inputs = ChatTemplateInputs { -// messages: vec![ -// Message { -// role: "user".to_string(), -// content: Some("Hi!".to_string()), -// name: None, -// tool_calls: None, -// }, -// Message { -// role: "user".to_string(), -// content: Some("Hi again!".to_string()), -// name: None, -// tool_calls: None, -// }, -// Message { -// role: "assistant".to_string(), -// content: Some("Hello how can I help?".to_string()), -// name: None, -// tool_calls: None, -// }, -// Message { -// role: "user".to_string(), -// content: Some("What is Deep Learning?".to_string()), -// name: None, -// tool_calls: None, -// }, -// Message { -// role: "assistant".to_string(), -// content: Some("magic!".to_string()), -// name: None, -// tool_calls: None, -// }, -// ], -// bos_token: Some("[BOS]"), -// eos_token: Some("[EOS]"), -// add_generation_prompt: true, -// ..Default::default() -// }; + let chat_template_inputs = ChatTemplateInputs { + messages: vec![ + Message { + role: "user".to_string(), + content: Some("Hi!".to_string()), + name: None, + tool_calls: None, + }, + Message { + role: "user".to_string(), + content: Some("Hi again!".to_string()), + name: None, + tool_calls: None, + }, + Message { + role: "assistant".to_string(), + content: Some("Hello how can I help?".to_string()), + name: None, + tool_calls: None, + }, + Message { + role: "user".to_string(), + content: Some("What is Deep Learning?".to_string()), + name: None, + tool_calls: None, + }, + Message { + role: "assistant".to_string(), + content: Some("magic!".to_string()), + name: None, + tool_calls: None, + }, + ], + bos_token: Some("[BOS]"), + eos_token: Some("[EOS]"), + add_generation_prompt: true, + ..Default::default() + }; -// let result = tmpl.unwrap().render(chat_template_inputs); //.err().unwrap(); + let result = tmpl.unwrap().render(chat_template_inputs); //.err().unwrap(); -// match result { -// Ok(_) => panic!("Should have failed"), -// Err(e) => { -// assert_eq!( -// e.detail().unwrap(), -// "Conversation roles must alternate user/assistant/user/assistant/..." -// ); -// } -// } -// } + match result { + Ok(_) => panic!("Should have failed"), + Err(e) => { + assert_eq!( + e.detail().unwrap(), + "Conversation roles must alternate user/assistant/user/assistant/..." + ); + } + } + } -// #[test] -// fn test_chat_template_valid_with_raise() { -// let mut env = Environment::new(); -// env.add_function("raise_exception", raise_exception); + #[test] + fn test_chat_template_valid_with_raise() { + let mut env = Environment::new(); + env.add_function("raise_exception", raise_exception); -// let source = r#" -// {{ bos_token }} -// {% for message in messages %} -// {% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %} -// {{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }} -// {% endif %} -// {% if message['role'] == 'user' %} -// {{ '[INST] ' + message['content'] + ' [/INST]' }} -// {% elif message['role'] == 'assistant' %} -// {{ message['content'] + eos_token}} -// {% else %} -// {{ raise_exception('Only user and assistant roles are supported!') }} -// {% endif %} -// {% endfor %}"#; + let source = r#" + {{ bos_token }} + {% for message in messages %} + {% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %} + {{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }} + {% endif %} + {% if message['role'] == 'user' %} + {{ '[INST] ' + message['content'] + ' [/INST]' }} + {% elif message['role'] == 'assistant' %} + {{ message['content'] + eos_token}} + {% else %} + {{ raise_exception('Only user and assistant roles are supported!') }} + {% endif %} + {% endfor %}"#; -// // trim all the whitespace -// let source = source -// .lines() -// .map(|line| line.trim()) -// .collect::>() -// .join(""); + // trim all the whitespace + let source = source + .lines() + .map(|line| line.trim()) + .collect::>() + .join(""); -// let tmpl = env.template_from_str(&source); + let tmpl = env.template_from_str(&source); -// let chat_template_inputs = ChatTemplateInputs { -// messages: vec![ -// Message { -// role: "user".to_string(), -// content: Some("Hi!".to_string()), -// name: None, -// tool_calls: None, -// }, -// Message { -// role: "assistant".to_string(), -// content: Some("Hello how can I help?".to_string()), -// name: None, -// tool_calls: None, -// }, -// Message { -// role: "user".to_string(), -// content: Some("What is Deep Learning?".to_string()), -// name: None, -// tool_calls: None, -// }, -// Message { -// role: "assistant".to_string(), -// content: Some("magic!".to_string()), -// name: None, -// tool_calls: None, -// }, -// ], -// bos_token: Some("[BOS]"), -// eos_token: Some("[EOS]"), -// add_generation_prompt: true, -// ..Default::default() -// }; + let chat_template_inputs = ChatTemplateInputs { + messages: vec![ + Message { + role: "user".to_string(), + content: Some("Hi!".to_string()), + name: None, + tool_calls: None, + }, + Message { + role: "assistant".to_string(), + content: Some("Hello how can I help?".to_string()), + name: None, + tool_calls: None, + }, + Message { + role: "user".to_string(), + content: Some("What is Deep Learning?".to_string()), + name: None, + tool_calls: None, + }, + Message { + role: "assistant".to_string(), + content: Some("magic!".to_string()), + name: None, + tool_calls: None, + }, + ], + bos_token: Some("[BOS]"), + eos_token: Some("[EOS]"), + add_generation_prompt: true, + ..Default::default() + }; -// let result = tmpl.unwrap().render(chat_template_inputs).unwrap(); -// assert_eq!(result, "[BOS][INST] Hi! [/INST]Hello how can I help?[EOS][INST] What is Deep Learning? [/INST]magic![EOS]"); -// } + let result = tmpl.unwrap().render(chat_template_inputs).unwrap(); + assert_eq!(result, "[BOS][INST] Hi! [/INST]Hello how can I help?[EOS][INST] What is Deep Learning? [/INST]magic![EOS]"); + } -// #[test] -// fn test_chat_template_valid_with_add_generation_prompt() { -// let mut env = Environment::new(); -// env.add_function("raise_exception", raise_exception); + #[test] + fn test_chat_template_valid_with_add_generation_prompt() { + let mut env = Environment::new(); + env.add_function("raise_exception", raise_exception); -// let source = r#" -// {% for message in messages %} -// {{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}} -// {% endfor %} -// {% if add_generation_prompt %} -// {{ '<|im_start|>assistant\n' }} -// {% endif %}"#; + let source = r#" + {% for message in messages %} + {{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}} + {% endfor %} + {% if add_generation_prompt %} + {{ '<|im_start|>assistant\n' }} + {% endif %}"#; -// // trim all the whitespace -// let source = source -// .lines() -// .map(|line| line.trim()) -// .collect::>() -// .join(""); + // trim all the whitespace + let source = source + .lines() + .map(|line| line.trim()) + .collect::>() + .join(""); -// let tmpl = env.template_from_str(&source); + let tmpl = env.template_from_str(&source); -// let chat_template_inputs = ChatTemplateInputs { -// messages: vec![ -// Message { -// role: "user".to_string(), -// content: Some("Hi!".to_string()), -// name: None, -// tool_calls: None, -// }, -// Message { -// role: "assistant".to_string(), -// content: Some("Hello how can I help?".to_string()), -// name: None, -// tool_calls: None, -// }, -// Message { -// role: "user".to_string(), -// content: Some("What is Deep Learning?".to_string()), -// name: None, -// tool_calls: None, -// }, -// Message { -// role: "assistant".to_string(), -// content: Some("magic!".to_string()), -// name: None, -// tool_calls: None, -// }, -// ], -// bos_token: Some("[BOS]"), -// eos_token: Some("[EOS]"), -// add_generation_prompt: true, -// ..Default::default() -// }; + let chat_template_inputs = ChatTemplateInputs { + messages: vec![ + Message { + role: "user".to_string(), + content: Some("Hi!".to_string()), + name: None, + tool_calls: None, + }, + Message { + role: "assistant".to_string(), + content: Some("Hello how can I help?".to_string()), + name: None, + tool_calls: None, + }, + Message { + role: "user".to_string(), + content: Some("What is Deep Learning?".to_string()), + name: None, + tool_calls: None, + }, + Message { + role: "assistant".to_string(), + content: Some("magic!".to_string()), + name: None, + tool_calls: None, + }, + ], + bos_token: Some("[BOS]"), + eos_token: Some("[EOS]"), + add_generation_prompt: true, + ..Default::default() + }; -// let result = tmpl.unwrap().render(chat_template_inputs).unwrap(); -// assert_eq!(result, "<|im_start|>user\nHi!<|im_end|>\n<|im_start|>assistant\nHello how can I help?<|im_end|>\n<|im_start|>user\nWhat is Deep Learning?<|im_end|>\n<|im_start|>assistant\nmagic!<|im_end|>\n<|im_start|>assistant\n"); -// } + let result = tmpl.unwrap().render(chat_template_inputs).unwrap(); + assert_eq!(result, "<|im_start|>user\nHi!<|im_end|>\n<|im_start|>assistant\nHello how can I help?<|im_end|>\n<|im_start|>user\nWhat is Deep Learning?<|im_end|>\n<|im_start|>assistant\nmagic!<|im_end|>\n<|im_start|>assistant\n"); + } -// struct ChatTemplateTestItem { -// name: &'static str, -// chat_template: &'static str, -// input: ChatTemplateInputs<'static>, -// target: &'static str, -// } + struct ChatTemplateTestItem { + name: &'static str, + chat_template: &'static str, + input: ChatTemplateInputs<'static>, + target: &'static str, + } -// #[test] -// fn test_many_chat_templates() { -// let example_chat = vec![ -// Message { -// role: "user".to_string(), -// content: Some("Hello, how are you?".to_string()), -// name: None, -// tool_calls: None, -// }, -// Message { -// role: "assistant".to_string(), -// content: Some("I'm doing great. How can I help you today?".to_string()), -// name: None, -// tool_calls: None, -// }, -// Message { -// role: "user".to_string(), -// content: Some("I'd like to show off how chat templating works!".to_string()), -// name: None, -// tool_calls: None, -// }, -// ]; + #[test] + fn test_many_chat_templates() { + let example_chat = vec![ + Message { + role: "user".to_string(), + content: Some("Hello, how are you?".to_string()), + name: None, + tool_calls: None, + }, + Message { + role: "assistant".to_string(), + content: Some("I'm doing great. How can I help you today?".to_string()), + name: None, + tool_calls: None, + }, + Message { + role: "user".to_string(), + content: Some("I'd like to show off how chat templating works!".to_string()), + name: None, + tool_calls: None, + }, + ]; -// let example_chat_with_system = vec![Message { -// role: "system".to_string(), -// content: Some( -// "You are a friendly chatbot who always responds in the style of a pirate" -// .to_string(), -// ), -// name: None, -// tool_calls: None, -// }] -// .iter() -// .chain(&example_chat) -// .cloned() -// .collect::>(); + let example_chat_with_system = vec![Message { + role: "system".to_string(), + content: Some( + "You are a friendly chatbot who always responds in the style of a pirate" + .to_string(), + ), + name: None, + tool_calls: None, + }] + .iter() + .chain(&example_chat) + .cloned() + .collect::>(); -// let test_default_templates = vec![ -// ChatTemplateTestItem { -// name: "_base", -// chat_template: "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\\n' + message['content'] + '<|im_end|>' + '\\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\\n' }}{% endif %}", -// input: ChatTemplateInputs { -// messages: example_chat.clone(), -// add_generation_prompt: false, -// bos_token: Some(""), -// eos_token: Some(""), -// ..Default::default() -// }, -// target: "<|im_start|>user\nHello, how are you?<|im_end|>\n<|im_start|>assistant\nI'm doing great. How can I help you today?<|im_end|>\n<|im_start|>user\nI'd like to show off how chat templating works!<|im_end|>\n", -// }, -// ChatTemplateTestItem { -// name: "blenderbot", -// chat_template: "{% for message in messages %}{% if message['role'] == 'user' %}{{ ' ' }}{% endif %}{{ message['content'] }}{% if not loop.last %}{{ ' ' }}{% endif %}{% endfor %}{{ eos_token }}", -// input: ChatTemplateInputs { -// messages: example_chat.clone(), -// add_generation_prompt: false, -// bos_token: Some(""), -// eos_token: Some(""), -// ..Default::default() -// }, -// target: " Hello, how are you? I'm doing great. How can I help you today? I'd like to show off how chat templating works!", -// }, -// ChatTemplateTestItem { -// name: "blenderbot_small", -// chat_template: "{% for message in messages %}{% if message['role'] == 'user' %}{{ ' ' }}{% endif %}{{ message['content'] }}{% if not loop.last %}{{ ' ' }}{% endif %}{% endfor %}{{ eos_token }}", -// input: ChatTemplateInputs { -// messages: example_chat.clone(), -// add_generation_prompt: false, -// bos_token: Some(""), -// eos_token: Some(""), -// ..Default::default() -// }, -// target: " Hello, how are you? I'm doing great. How can I help you today? I'd like to show off how chat templating works!", -// }, -// ChatTemplateTestItem { -// name: "bloom", -// chat_template: "{% for message in messages %}{{ message.content }}{{ eos_token }}{% endfor %}", -// input: ChatTemplateInputs { -// messages: example_chat.clone(), -// add_generation_prompt: false, -// bos_token: Some(""), -// eos_token: Some(""), -// ..Default::default() -// }, -// target: "Hello, how are you?I'm doing great. How can I help you today?I'd like to show off how chat templating works!", -// }, -// ChatTemplateTestItem { -// name: "gpt_neox", -// chat_template: "{% for message in messages %}{{ message.content }}{{ eos_token }}{% endfor %}", -// input: ChatTemplateInputs { -// messages: example_chat.clone(), -// add_generation_prompt: false, -// bos_token: Some(""), -// eos_token: Some("<|endoftext|>"), -// ..Default::default() -// }, -// target: "Hello, how are you?<|endoftext|>I'm doing great. How can I help you today?<|endoftext|>I'd like to show off how chat templating works!<|endoftext|>", -// }, -// ChatTemplateTestItem { -// name: "gpt2", -// chat_template: "{% for message in messages %}{{ message.content }}{{ eos_token }}{% endfor %}", -// input: ChatTemplateInputs { -// messages: example_chat.clone(), -// add_generation_prompt: false, -// bos_token: Some(""), -// eos_token: Some("<|endoftext|>"), -// ..Default::default() -// }, -// target: "Hello, how are you?<|endoftext|>I'm doing great. How can I help you today?<|endoftext|>I'd like to show off how chat templating works!<|endoftext|>", -// }, -// ChatTemplateTestItem { -// name: "llama", -// // NOTE: the `.strip()` has been replaced with `| trim` in the following template -// chat_template: "{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% elif USE_DEFAULT_PROMPT == true and not '<>' in messages[0]['content'] %}{% set loop_messages = messages %}{% set system_message = 'DEFAULT_SYSTEM_MESSAGE' %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 and system_message != false %}{% set content = '<>\\n' + system_message + '\\n<>\\n\\n' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ bos_token +'[INST] ' + content | trim + ' [/INST]' }}{% elif message['role'] == 'system' %}{{ '<>\\n' + content | trim + '\\n<>\\n\\n' }}{% elif message['role'] == 'assistant' %}{{ ' ' + content | trim + ' ' + eos_token }}{% endif %}{% endfor %}", -// input: ChatTemplateInputs { -// messages: example_chat_with_system.clone(), -// add_generation_prompt: true, -// bos_token: Some(""), -// eos_token: Some(""), -// ..Default::default() -// }, -// target: "[INST] <>\nYou are a friendly chatbot who always responds in the style of a pirate\n<>\n\nHello, how are you? [/INST] I'm doing great. How can I help you today? [INST] I'd like to show off how chat templating works! [/INST]", -// }, -// ChatTemplateTestItem { -// name: "whisper", -// chat_template: "{% for message in messages %}{{ message.content }}{{ eos_token }}{% endfor %}", -// input: ChatTemplateInputs { -// messages: example_chat.clone(), -// add_generation_prompt: true, -// bos_token: Some(""), -// eos_token: Some("<|endoftext|>"), -// ..Default::default() -// }, -// target: "Hello, how are you?<|endoftext|>I'm doing great. How can I help you today?<|endoftext|>I'd like to show off how chat templating works!<|endoftext|>", -// }, -// ]; + let test_default_templates = vec![ + ChatTemplateTestItem { + name: "_base", + chat_template: "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\\n' + message['content'] + '<|im_end|>' + '\\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\\n' }}{% endif %}", + input: ChatTemplateInputs { + messages: example_chat.clone(), + add_generation_prompt: false, + bos_token: Some(""), + eos_token: Some(""), + ..Default::default() + }, + target: "<|im_start|>user\nHello, how are you?<|im_end|>\n<|im_start|>assistant\nI'm doing great. How can I help you today?<|im_end|>\n<|im_start|>user\nI'd like to show off how chat templating works!<|im_end|>\n", + }, + ChatTemplateTestItem { + name: "blenderbot", + chat_template: "{% for message in messages %}{% if message['role'] == 'user' %}{{ ' ' }}{% endif %}{{ message['content'] }}{% if not loop.last %}{{ ' ' }}{% endif %}{% endfor %}{{ eos_token }}", + input: ChatTemplateInputs { + messages: example_chat.clone(), + add_generation_prompt: false, + bos_token: Some(""), + eos_token: Some(""), + ..Default::default() + }, + target: " Hello, how are you? I'm doing great. How can I help you today? I'd like to show off how chat templating works!", + }, + ChatTemplateTestItem { + name: "blenderbot_small", + chat_template: "{% for message in messages %}{% if message['role'] == 'user' %}{{ ' ' }}{% endif %}{{ message['content'] }}{% if not loop.last %}{{ ' ' }}{% endif %}{% endfor %}{{ eos_token }}", + input: ChatTemplateInputs { + messages: example_chat.clone(), + add_generation_prompt: false, + bos_token: Some(""), + eos_token: Some(""), + ..Default::default() + }, + target: " Hello, how are you? I'm doing great. How can I help you today? I'd like to show off how chat templating works!", + }, + ChatTemplateTestItem { + name: "bloom", + chat_template: "{% for message in messages %}{{ message.content }}{{ eos_token }}{% endfor %}", + input: ChatTemplateInputs { + messages: example_chat.clone(), + add_generation_prompt: false, + bos_token: Some(""), + eos_token: Some(""), + ..Default::default() + }, + target: "Hello, how are you?I'm doing great. How can I help you today?I'd like to show off how chat templating works!", + }, + ChatTemplateTestItem { + name: "gpt_neox", + chat_template: "{% for message in messages %}{{ message.content }}{{ eos_token }}{% endfor %}", + input: ChatTemplateInputs { + messages: example_chat.clone(), + add_generation_prompt: false, + bos_token: Some(""), + eos_token: Some("<|endoftext|>"), + ..Default::default() + }, + target: "Hello, how are you?<|endoftext|>I'm doing great. How can I help you today?<|endoftext|>I'd like to show off how chat templating works!<|endoftext|>", + }, + ChatTemplateTestItem { + name: "gpt2", + chat_template: "{% for message in messages %}{{ message.content }}{{ eos_token }}{% endfor %}", + input: ChatTemplateInputs { + messages: example_chat.clone(), + add_generation_prompt: false, + bos_token: Some(""), + eos_token: Some("<|endoftext|>"), + ..Default::default() + }, + target: "Hello, how are you?<|endoftext|>I'm doing great. How can I help you today?<|endoftext|>I'd like to show off how chat templating works!<|endoftext|>", + }, + ChatTemplateTestItem { + name: "llama", + // NOTE: the `.strip()` has been replaced with `| trim` in the following template + chat_template: "{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% elif USE_DEFAULT_PROMPT == true and not '<>' in messages[0]['content'] %}{% set loop_messages = messages %}{% set system_message = 'DEFAULT_SYSTEM_MESSAGE' %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 and system_message != false %}{% set content = '<>\\n' + system_message + '\\n<>\\n\\n' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ bos_token +'[INST] ' + content | trim + ' [/INST]' }}{% elif message['role'] == 'system' %}{{ '<>\\n' + content | trim + '\\n<>\\n\\n' }}{% elif message['role'] == 'assistant' %}{{ ' ' + content | trim + ' ' + eos_token }}{% endif %}{% endfor %}", + input: ChatTemplateInputs { + messages: example_chat_with_system.clone(), + add_generation_prompt: true, + bos_token: Some(""), + eos_token: Some(""), + ..Default::default() + }, + target: "[INST] <>\nYou are a friendly chatbot who always responds in the style of a pirate\n<>\n\nHello, how are you? [/INST] I'm doing great. How can I help you today? [INST] I'd like to show off how chat templating works! [/INST]", + }, + ChatTemplateTestItem { + name: "whisper", + chat_template: "{% for message in messages %}{{ message.content }}{{ eos_token }}{% endfor %}", + input: ChatTemplateInputs { + messages: example_chat.clone(), + add_generation_prompt: true, + bos_token: Some(""), + eos_token: Some("<|endoftext|>"), + ..Default::default() + }, + target: "Hello, how are you?<|endoftext|>I'm doing great. How can I help you today?<|endoftext|>I'd like to show off how chat templating works!<|endoftext|>", + }, + ]; -// #[allow(unused_variables)] // name is unused -// for ChatTemplateTestItem { -// name, -// chat_template, -// input, -// target, -// } in test_default_templates -// { -// let mut env = Environment::new(); -// env.add_function("raise_exception", raise_exception); -// let tmpl = env.template_from_str(&chat_template); -// let result = tmpl.unwrap().render(input).unwrap(); -// assert_eq!(result, target); -// } + #[allow(unused_variables)] // name is unused + for ChatTemplateTestItem { + name, + chat_template, + input, + target, + } in test_default_templates + { + let mut env = Environment::new(); + env.add_function("raise_exception", raise_exception); + let tmpl = env.template_from_str(&chat_template); + let result = tmpl.unwrap().render(input).unwrap(); + assert_eq!(result, target); + } -// let test_custom_templates = vec![ -// ChatTemplateTestItem { -// name: "HuggingFaceH4/zephyr-7b-beta (add_generation_prompt=false)", -// chat_template: "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\\n' + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}", -// input: ChatTemplateInputs { -// messages: example_chat_with_system.clone(), -// add_generation_prompt: false, -// bos_token: Some(""), -// eos_token: Some(""), -// ..Default::default() -// }, -// target: "<|system|>\nYou are a friendly chatbot who always responds in the style of a pirate<|user|>\nHello, how are you?<|assistant|>\nI'm doing great. How can I help you today?<|user|>\nI'd like to show off how chat templating works!", -// }, -// ChatTemplateTestItem { -// name: "HuggingFaceH4/zephyr-7b-beta (add_generation_prompt=true)", -// chat_template: "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\\n' + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}", -// input: ChatTemplateInputs { -// messages: vec![ -// Message { -// role: "system".to_string(), -// content: Some("You are a friendly chatbot who always responds in the style of a pirate".to_string()), -// name: None, -// tool_calls: None, -// }, -// Message { -// role: "user".to_string(), -// content: Some("How many helicopters can a human eat in one sitting?".to_string()), -// name: None, -// tool_calls: None, -// }, -// ], -// add_generation_prompt: true, -// bos_token: Some(""), -// eos_token: Some(""), -// ..Default::default() -// }, -// target: "<|system|>\nYou are a friendly chatbot who always responds in the style of a pirate<|user|>\nHow many helicopters can a human eat in one sitting?<|assistant|>", -// }, -// ChatTemplateTestItem { -// name: "HuggingFaceH4/zephyr-7b-gemma-v0.1", -// chat_template: "{% if messages[0]['role'] == 'user' or messages[0]['role'] == 'system' %}{{ bos_token }}{% endif %}{% for message in messages %}{{ '<|im_start|>' + message['role'] + '\\n' + message['content'] + '<|im_end|>' + '\\n' }}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% elif messages[-1]['role'] == 'assistant' %}{{ eos_token }}{% endif %}", -// input: ChatTemplateInputs { -// messages: example_chat.clone(), -// add_generation_prompt: false, -// bos_token: Some(""), -// eos_token: Some(""), -// ..Default::default() -// }, -// target: "<|im_start|>user\nHello, how are you?<|im_end|>\n<|im_start|>assistant\nI'm doing great. How can I help you today?<|im_end|>\n<|im_start|>user\nI'd like to show off how chat templating works!<|im_end|>\n", -// }, -// ChatTemplateTestItem { -// name: "mistralai/Mistral-7B-Instruct-v0.1", -// chat_template: "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token + ' ' }}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %}", -// input: ChatTemplateInputs { -// messages: example_chat.clone(), -// add_generation_prompt: false, -// bos_token: Some(""), -// eos_token: Some(""), -// ..Default::default() -// }, -// target: "[INST] Hello, how are you? [/INST]I'm doing great. How can I help you today? [INST] I'd like to show off how chat templating works! [/INST]", -// }, -// ChatTemplateTestItem { -// name: "mistralai/Mixtral-8x7B-Instruct-v0.1", -// chat_template: "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token}}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %}", -// input: ChatTemplateInputs { -// messages: example_chat.clone(), -// add_generation_prompt: false, -// bos_token: Some(""), -// eos_token: Some(""), -// ..Default::default() -// }, -// target: "[INST] Hello, how are you? [/INST]I'm doing great. How can I help you today?[INST] I'd like to show off how chat templating works! [/INST]", -// }, -// ChatTemplateTestItem { -// name: "cognitivecomputations/dolphin-2.5-mixtral-8x7b", -// chat_template: "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\\n' + message['content'] + '<|im_end|>' + '\\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\\n' }}{% endif %}", -// input: ChatTemplateInputs { -// messages: example_chat.clone(), -// add_generation_prompt: false, -// bos_token: Some(""), -// eos_token: Some(""), -// ..Default::default() -// }, -// target: "<|im_start|>user\nHello, how are you?<|im_end|>\n<|im_start|>assistant\nI'm doing great. How can I help you today?<|im_end|>\n<|im_start|>user\nI'd like to show off how chat templating works!<|im_end|>\n", -// }, -// ChatTemplateTestItem { -// name: "openchat/openchat-3.5-0106", -// // `.title()` has been replaced with `| upper` in the following template -// chat_template: "{{ bos_token }}{% for message in messages %}{{ 'GPT4 Correct ' + (message['role'] | title) + ': ' + message['content'] + '<|end_of_turn|>'}}{% endfor %}{% if add_generation_prompt %}{{ 'GPT4 Correct Assistant:' }}{% endif %}", -// input: ChatTemplateInputs { -// messages: example_chat.clone(), -// add_generation_prompt: false, -// bos_token: Some(""), -// eos_token: Some(""), -// ..Default::default() -// }, -// target: "GPT4 Correct User: Hello, how are you?<|end_of_turn|>GPT4 Correct Assistant: I'm doing great. How can I help you today?<|end_of_turn|>GPT4 Correct User: I'd like to show off how chat templating works!<|end_of_turn|>", -// }, -// ChatTemplateTestItem { -// name: "upstage/SOLAR-10.7B-Instruct-v1.0", -// chat_template: "{% for message in messages %}{{ message.content }}{{ eos_token }}{% endfor %}", -// input: ChatTemplateInputs { -// messages: example_chat.clone(), -// add_generation_prompt: false, -// bos_token: Some(""), -// eos_token: Some(""), -// ..Default::default() -// }, -// target: "Hello, how are you?I'm doing great. How can I help you today?I'd like to show off how chat templating works!", -// }, -// ChatTemplateTestItem { -// name: "codellama/CodeLlama-70b-Instruct-hf", -// // NOTE: `.strip()` has been replaced with `| trim` in the following template -// chat_template: "{% if messages[0]['role'] == 'system' %}{% set user_index = 1 %}{% else %}{% set user_index = 0 %}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != ((loop.index0 + user_index) % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 %}{{ '' }}{% endif %}{% set content = 'Source: ' + message['role'] + '\\n\\n ' + message['content'] | trim %}{{ content + ' ' }}{% endfor %}{{'Source: assistant\\nDestination: user\\n\\n '}}", -// input: ChatTemplateInputs { -// messages: example_chat.clone(), -// add_generation_prompt: false, -// bos_token: Some(""), -// eos_token: Some(""), -// ..Default::default() -// }, -// target: "Source: user\n\n Hello, how are you? Source: assistant\n\n I'm doing great. How can I help you today? Source: user\n\n I'd like to show off how chat templating works! Source: assistant\nDestination: user\n\n ", -// }, -// ChatTemplateTestItem { -// name: "Deci/DeciLM-7B-instruct", -// chat_template: "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '### User:\\n' + message['content'] }}\n{% elif message['role'] == 'system' %}\n{{ '### System:\\n' + message['content'] }}\n{% elif message['role'] == 'assistant' %}\n{{ '### Assistant:\\n' + message['content'] }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '### Assistant:' }}\n{% endif %}\n{% endfor %}", -// input: ChatTemplateInputs { -// messages: example_chat.clone(), -// add_generation_prompt: false, -// bos_token: Some(""), -// eos_token: Some(""), -// ..Default::default() -// }, -// target: "### User:\nHello, how are you?### Assistant:\nI'm doing great. How can I help you today?### User:\nI'd like to show off how chat templating works!", -// }, -// ChatTemplateTestItem { -// name: "Qwen/Qwen1.5-72B-Chat", -// chat_template: "{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system\\nYou are a helpful assistant<|im_end|>\\n' }}{% endif %}{{'<|im_start|>' + message['role'] + '\\n' + message['content']}}{% if (loop.last and add_generation_prompt) or not loop.last %}{{ '<|im_end|>' + '\\n'}}{% endif %}{% endfor %}{% if add_generation_prompt and messages[-1]['role'] != 'assistant' %}{{ '<|im_start|>assistant\\n' }}{% endif %}", -// input: ChatTemplateInputs { -// messages: example_chat.clone(), -// add_generation_prompt: false, -// bos_token: Some(""), -// eos_token: Some(""), -// ..Default::default() -// }, -// target: "<|im_start|>system\nYou are a helpful assistant<|im_end|>\n<|im_start|>user\nHello, how are you?<|im_end|>\n<|im_start|>assistant\nI'm doing great. How can I help you today?<|im_end|>\n<|im_start|>user\nI'd like to show off how chat templating works!", -// }, -// ChatTemplateTestItem { -// name: "deepseek-ai/deepseek-llm-7b-chat", -// chat_template: "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{{ bos_token }}{% for message in messages %}{% if message['role'] == 'user' %}{{ 'User: ' + message['content'] + '\\n\\n' }}{% elif message['role'] == 'assistant' %}{{ 'Assistant: ' + message['content'] + eos_token }}{% elif message['role'] == 'system' %}{{ message['content'] + '\\n\\n' }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ 'Assistant:' }}{% endif %}", -// input: ChatTemplateInputs { -// messages: example_chat.clone(), -// add_generation_prompt: false, -// bos_token: Some("<｜begin▁of▁sentence｜>"), -// eos_token: Some("<｜end▁of▁sentence｜>"), -// ..Default::default() -// }, -// target: "<｜begin▁of▁sentence｜>User: Hello, how are you?\n\nAssistant: I'm doing great. How can I help you today?<｜end▁of▁sentence｜>User: I'd like to show off how chat templating works!\n\n", -// }, -// ChatTemplateTestItem { -// name: "h2oai/h2o-danube-1.8b-chat", -// chat_template: "{% for message in messages %}{% if message['role'] == 'user' %}{{ '<|prompt|>' + message['content'] + eos_token }}{% elif message['role'] == 'system' %}{{ '<|system|>' + message['content'] + eos_token }}{% elif message['role'] == 'assistant' %}{{ '<|answer|>' + message['content'] + eos_token }}{% endif %}{% if loop.last and add_generation_prompt %}{{ '<|answer|>' }}{% endif %}{% endfor %}", -// input: ChatTemplateInputs { -// messages: example_chat.clone(), -// add_generation_prompt: false, -// bos_token: Some(""), -// eos_token: Some(""), -// ..Default::default() -// }, -// target: "<|prompt|>Hello, how are you?<|answer|>I'm doing great. How can I help you today?<|prompt|>I'd like to show off how chat templating works!", -// }, -// ChatTemplateTestItem { -// name: "internlm/internlm2-chat-7b", -// chat_template: "{% if messages[0]['role'] == 'user' or messages[0]['role'] == 'system' %}{{ bos_token }}{% endif %}{% for message in messages %}{{ '<|im_start|>' + message['role'] + '\\n' + message['content'] + '<|im_end|>' + '\\n' }}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\\n' }}{% elif messages[-1]['role'] == 'assistant' %}{{ eos_token }}{% endif %}", -// input: ChatTemplateInputs { -// messages: example_chat.clone(), -// add_generation_prompt: false, -// bos_token: Some(""), -// eos_token: Some(""), -// ..Default::default() -// }, -// target: "<|im_start|>user\nHello, how are you?<|im_end|>\n<|im_start|>assistant\nI'm doing great. How can I help you today?<|im_end|>\n<|im_start|>user\nI'd like to show off how chat templating works!<|im_end|>\n", -// }, -// ChatTemplateTestItem { -// name: "TheBloke/deepseek-coder-33B-instruct-AWQ", -// chat_template: "{%- set found_item = false -%}\n{%- for message in messages -%}\n {%- if message['role'] == 'system' -%}\n {%- set found_item = true -%}\n {%- endif -%}\n{%- endfor -%}\n{%- if not found_item -%}\n{{'You are an AI programming assistant, utilizing the Deepseek Coder model, developed by Deepseek Company, and you only answer questions related to computer science. For politically sensitive questions, security and privacy issues, and other non-computer science questions, you will refuse to answer.\\n'}}\n{%- endif %}\n{%- for message in messages %}\n {%- if message['role'] == 'system' %}\n{{ message['content'] }}\n {%- else %}\n {%- if message['role'] == 'user' %}\n{{'### Instruction:\\n' + message['content'] + '\\n'}}\n {%- else %}\n{{'### Response:\\n' + message['content'] + '\\n<|EOT|>\\n'}}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{{'### Response:\\n'}}\n", -// input: ChatTemplateInputs { -// messages: example_chat.clone(), -// add_generation_prompt: false, -// bos_token: Some("<｜begin▁of▁sentence｜>"), -// eos_token: Some("<|EOT|>"), -// ..Default::default() -// }, -// target: "You are an AI programming assistant, utilizing the Deepseek Coder model, developed by Deepseek Company, and you only answer questions related to computer science. For politically sensitive questions, security and privacy issues, and other non-computer science questions, you will refuse to answer.\n### Instruction:\nHello, how are you?\n### Response:\nI'm doing great. How can I help you today?\n<|EOT|>\n### Instruction:\nI'd like to show off how chat templating works!\n### Response:\n", -// }, -// ChatTemplateTestItem { -// name: "ericzzz/falcon-rw-1b-chat", -// // `.strip()` has been replaced with `| trim` in the following template -// chat_template: "{% for message in messages %}{% if loop.index > 1 and loop.previtem['role'] != 'assistant' %}{{ ' ' }}{% endif %}{% if message['role'] == 'system' %}{{ '[SYS] ' + message['content'] | trim }}{% elif message['role'] == 'user' %}{{ '[INST] ' + message['content'] | trim }}{% elif message['role'] == 'assistant' %}{{ '[RESP] ' + message['content'] + eos_token }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ ' [RESP] ' }}{% endif %}", -// input: ChatTemplateInputs { -// messages: example_chat.clone(), -// add_generation_prompt: false, -// bos_token: Some("<|endoftext|>"), -// eos_token: Some("<|endoftext|>"), -// ..Default::default() -// }, -// target: "[INST] Hello, how are you? [RESP] I'm doing great. How can I help you today?<|endoftext|>[INST] I'd like to show off how chat templating works!", -// }, -// ChatTemplateTestItem { -// name: "abacusai/Smaug-34B-v0.1", -// chat_template: "{%- for idx in range(0, messages|length) -%}\n{%- if messages[idx]['role'] == 'user' -%}\n{%- if idx > 1 -%}\n{{- bos_token + '[INST] ' + messages[idx]['content'] + ' [/INST]' -}}\n{%- else -%}\n{{- messages[idx]['content'] + ' [/INST]' -}}\n{%- endif -%}\n{% elif messages[idx]['role'] == 'system' %}\n{{- '[INST] <>\\n' + messages[idx]['content'] + '\\n<>\\n\\n' -}}\n{%- elif messages[idx]['role'] == 'assistant' -%}\n{{- ' ' + messages[idx]['content'] + ' ' + eos_token -}}\n{% endif %}\n{% endfor %}", -// input: ChatTemplateInputs { -// messages: example_chat.clone(), -// add_generation_prompt: false, -// bos_token: Some(""), -// eos_token: Some(""), -// ..Default::default() -// }, -// target: "Hello, how are you? [/INST] I'm doing great. How can I help you today? [INST] I'd like to show off how chat templating works! [/INST]", -// }, -// ChatTemplateTestItem { -// name: "maywell/Synatra-Mixtral-8x7B", -// chat_template: "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n{% for message in messages %}{% if message['role'] == 'user' %}### Instruction:\n{{ message['content']|trim -}}{% if not loop.last %}{% endif %}\n{% elif message['role'] == 'assistant' %}### Response:\n{{ message['content']|trim -}}{% if not loop.last %}{% endif %}\n{% elif message['role'] == 'system' %}{{ message['content']|trim -}}{% if not loop.last %}{% endif %}\n{% endif %}\n{% endfor %}\n{% if add_generation_prompt and messages[-1]['role'] != 'assistant' %}\n### Response:\n{% endif %}", -// input: ChatTemplateInputs { -// messages: example_chat.clone(), -// add_generation_prompt: false, -// bos_token: Some(""), -// eos_token: Some(""), -// ..Default::default() -// }, -// target: "Below is an instruction that describes a task. Write a response that appropriately completes the request.### Instruction:Hello, how are you?### Response:I'm doing great. How can I help you today?### Instruction:I'd like to show off how chat templating works!", -// }, -// ChatTemplateTestItem { -// name: "deepseek-ai/deepseek-coder-33b-instruct", -// chat_template: "{% if not add_generation_prompt is defined %}\n{% set add_generation_prompt = false %}\n{% endif %}\n{%- set ns = namespace(found=false) -%}\n{%- for message in messages -%}\n {%- if message['role'] == 'system' -%}\n {%- set ns.found = true -%}\n {%- endif -%}\n{%- endfor -%}\n{{bos_token}}{%- if not ns.found -%}\n{{'You are an AI programming assistant, utilizing the Deepseek Coder model, developed by Deepseek Company, and you only answer questions related to computer science. For politically sensitive questions, security and privacy issues, and other non-computer science questions, you will refuse to answer\\n'}}\n{%- endif %}\n{%- for message in messages %}\n {%- if message['role'] == 'system' %}\n{{ message['content'] }}\n {%- else %}\n {%- if message['role'] == 'user' %}\n{{'### Instruction:\\n' + message['content'] + '\\n'}}\n {%- else %}\n{{'### Response:\\n' + message['content'] + '\\n<|EOT|>\\n'}}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{% if add_generation_prompt %}\n{{'### Response:'}}\n{% endif %}", -// input: ChatTemplateInputs { -// messages: example_chat.clone(), -// add_generation_prompt: false, -// bos_token: Some("<｜begin▁of▁sentence｜>"), -// eos_token: Some(""), -// ..Default::default() -// }, -// target: "<｜begin▁of▁sentence｜>You are an AI programming assistant, utilizing the Deepseek Coder model, developed by Deepseek Company, and you only answer questions related to computer science. For politically sensitive questions, security and privacy issues, and other non-computer science questions, you will refuse to answer\n### Instruction:\nHello, how are you?\n### Response:\nI'm doing great. How can I help you today?\n<|EOT|>\n### Instruction:\nI'd like to show off how chat templating works!\n", -// }, -// // NOT INCLUDED -// // - meetkai/functionary-medium-v2.2 -// // - fireworks-ai/firefunction-v1 -// // https://github -// ChatTemplateTestItem { -// name: "maywell/PiVoT-MoE", -// chat_template: "{{ (messages|selectattr('role', 'equalto', 'system')|list|last).content|trim if (messages|selectattr('role', 'equalto', 'system')|list) else '' }}{% for message in messages %}{% if message['role'] == 'system' %}{{ message['content']|trim }}{% elif message['role'] == 'user' %}### Instruction: {{ message['content']|trim }}{% elif message['role'] == 'assistant' %}### Response: {{ message['content']|trim }}{% elif message['role'] == 'user_context' %}### Input: {{ message['content']|trim }}{% endif %}{% if not loop.last %}\n{% endif %}{% endfor %}{% if add_generation_prompt and messages[-1]['role'] != 'assistant' %}### Response:{% endif %}", -// input: ChatTemplateInputs { -// messages: example_chat_with_system.clone(), -// add_generation_prompt: false, -// bos_token: Some(""), -// eos_token: Some(""), -// ..Default::default() -// }, -// target: "You are a friendly chatbot who always responds in the style of a pirateYou are a friendly chatbot who always responds in the style of a pirate### Instruction: Hello, how are you?### Response: I'm doing great. How can I help you today?### Instruction: I'd like to show off how chat templating works!", -// }, -// ]; + let test_custom_templates = vec![ + ChatTemplateTestItem { + name: "HuggingFaceH4/zephyr-7b-beta (add_generation_prompt=false)", + chat_template: "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\\n' + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}", + input: ChatTemplateInputs { + messages: example_chat_with_system.clone(), + add_generation_prompt: false, + bos_token: Some(""), + eos_token: Some(""), + ..Default::default() + }, + target: "<|system|>\nYou are a friendly chatbot who always responds in the style of a pirate<|user|>\nHello, how are you?<|assistant|>\nI'm doing great. How can I help you today?<|user|>\nI'd like to show off how chat templating works!", + }, + ChatTemplateTestItem { + name: "HuggingFaceH4/zephyr-7b-beta (add_generation_prompt=true)", + chat_template: "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\\n' + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}", + input: ChatTemplateInputs { + messages: vec![ + Message { + role: "system".to_string(), + content: Some("You are a friendly chatbot who always responds in the style of a pirate".to_string()), + name: None, + tool_calls: None, + }, + Message { + role: "user".to_string(), + content: Some("How many helicopters can a human eat in one sitting?".to_string()), + name: None, + tool_calls: None, + }, + ], + add_generation_prompt: true, + bos_token: Some(""), + eos_token: Some(""), + ..Default::default() + }, + target: "<|system|>\nYou are a friendly chatbot who always responds in the style of a pirate<|user|>\nHow many helicopters can a human eat in one sitting?<|assistant|>", + }, + ChatTemplateTestItem { + name: "HuggingFaceH4/zephyr-7b-gemma-v0.1", + chat_template: "{% if messages[0]['role'] == 'user' or messages[0]['role'] == 'system' %}{{ bos_token }}{% endif %}{% for message in messages %}{{ '<|im_start|>' + message['role'] + '\\n' + message['content'] + '<|im_end|>' + '\\n' }}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% elif messages[-1]['role'] == 'assistant' %}{{ eos_token }}{% endif %}", + input: ChatTemplateInputs { + messages: example_chat.clone(), + add_generation_prompt: false, + bos_token: Some(""), + eos_token: Some(""), + ..Default::default() + }, + target: "<|im_start|>user\nHello, how are you?<|im_end|>\n<|im_start|>assistant\nI'm doing great. How can I help you today?<|im_end|>\n<|im_start|>user\nI'd like to show off how chat templating works!<|im_end|>\n", + }, + ChatTemplateTestItem { + name: "mistralai/Mistral-7B-Instruct-v0.1", + chat_template: "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token + ' ' }}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %}", + input: ChatTemplateInputs { + messages: example_chat.clone(), + add_generation_prompt: false, + bos_token: Some(""), + eos_token: Some(""), + ..Default::default() + }, + target: "[INST] Hello, how are you? [/INST]I'm doing great. How can I help you today? [INST] I'd like to show off how chat templating works! [/INST]", + }, + ChatTemplateTestItem { + name: "mistralai/Mixtral-8x7B-Instruct-v0.1", + chat_template: "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token}}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %}", + input: ChatTemplateInputs { + messages: example_chat.clone(), + add_generation_prompt: false, + bos_token: Some(""), + eos_token: Some(""), + ..Default::default() + }, + target: "[INST] Hello, how are you? [/INST]I'm doing great. How can I help you today?[INST] I'd like to show off how chat templating works! [/INST]", + }, + ChatTemplateTestItem { + name: "cognitivecomputations/dolphin-2.5-mixtral-8x7b", + chat_template: "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\\n' + message['content'] + '<|im_end|>' + '\\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\\n' }}{% endif %}", + input: ChatTemplateInputs { + messages: example_chat.clone(), + add_generation_prompt: false, + bos_token: Some(""), + eos_token: Some(""), + ..Default::default() + }, + target: "<|im_start|>user\nHello, how are you?<|im_end|>\n<|im_start|>assistant\nI'm doing great. How can I help you today?<|im_end|>\n<|im_start|>user\nI'd like to show off how chat templating works!<|im_end|>\n", + }, + ChatTemplateTestItem { + name: "openchat/openchat-3.5-0106", + // `.title()` has been replaced with `| upper` in the following template + chat_template: "{{ bos_token }}{% for message in messages %}{{ 'GPT4 Correct ' + (message['role'] | title) + ': ' + message['content'] + '<|end_of_turn|>'}}{% endfor %}{% if add_generation_prompt %}{{ 'GPT4 Correct Assistant:' }}{% endif %}", + input: ChatTemplateInputs { + messages: example_chat.clone(), + add_generation_prompt: false, + bos_token: Some(""), + eos_token: Some(""), + ..Default::default() + }, + target: "GPT4 Correct User: Hello, how are you?<|end_of_turn|>GPT4 Correct Assistant: I'm doing great. How can I help you today?<|end_of_turn|>GPT4 Correct User: I'd like to show off how chat templating works!<|end_of_turn|>", + }, + ChatTemplateTestItem { + name: "upstage/SOLAR-10.7B-Instruct-v1.0", + chat_template: "{% for message in messages %}{{ message.content }}{{ eos_token }}{% endfor %}", + input: ChatTemplateInputs { + messages: example_chat.clone(), + add_generation_prompt: false, + bos_token: Some(""), + eos_token: Some(""), + ..Default::default() + }, + target: "Hello, how are you?I'm doing great. How can I help you today?I'd like to show off how chat templating works!", + }, + ChatTemplateTestItem { + name: "codellama/CodeLlama-70b-Instruct-hf", + // NOTE: `.strip()` has been replaced with `| trim` in the following template + chat_template: "{% if messages[0]['role'] == 'system' %}{% set user_index = 1 %}{% else %}{% set user_index = 0 %}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != ((loop.index0 + user_index) % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 %}{{ '' }}{% endif %}{% set content = 'Source: ' + message['role'] + '\\n\\n ' + message['content'] | trim %}{{ content + ' ' }}{% endfor %}{{'Source: assistant\\nDestination: user\\n\\n '}}", + input: ChatTemplateInputs { + messages: example_chat.clone(), + add_generation_prompt: false, + bos_token: Some(""), + eos_token: Some(""), + ..Default::default() + }, + target: "Source: user\n\n Hello, how are you? Source: assistant\n\n I'm doing great. How can I help you today? Source: user\n\n I'd like to show off how chat templating works! Source: assistant\nDestination: user\n\n ", + }, + ChatTemplateTestItem { + name: "Deci/DeciLM-7B-instruct", + chat_template: "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '### User:\\n' + message['content'] }}\n{% elif message['role'] == 'system' %}\n{{ '### System:\\n' + message['content'] }}\n{% elif message['role'] == 'assistant' %}\n{{ '### Assistant:\\n' + message['content'] }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '### Assistant:' }}\n{% endif %}\n{% endfor %}", + input: ChatTemplateInputs { + messages: example_chat.clone(), + add_generation_prompt: false, + bos_token: Some(""), + eos_token: Some(""), + ..Default::default() + }, + target: "### User:\nHello, how are you?### Assistant:\nI'm doing great. How can I help you today?### User:\nI'd like to show off how chat templating works!", + }, + ChatTemplateTestItem { + name: "Qwen/Qwen1.5-72B-Chat", + chat_template: "{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system\\nYou are a helpful assistant<|im_end|>\\n' }}{% endif %}{{'<|im_start|>' + message['role'] + '\\n' + message['content']}}{% if (loop.last and add_generation_prompt) or not loop.last %}{{ '<|im_end|>' + '\\n'}}{% endif %}{% endfor %}{% if add_generation_prompt and messages[-1]['role'] != 'assistant' %}{{ '<|im_start|>assistant\\n' }}{% endif %}", + input: ChatTemplateInputs { + messages: example_chat.clone(), + add_generation_prompt: false, + bos_token: Some(""), + eos_token: Some(""), + ..Default::default() + }, + target: "<|im_start|>system\nYou are a helpful assistant<|im_end|>\n<|im_start|>user\nHello, how are you?<|im_end|>\n<|im_start|>assistant\nI'm doing great. How can I help you today?<|im_end|>\n<|im_start|>user\nI'd like to show off how chat templating works!", + }, + ChatTemplateTestItem { + name: "deepseek-ai/deepseek-llm-7b-chat", + chat_template: "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{{ bos_token }}{% for message in messages %}{% if message['role'] == 'user' %}{{ 'User: ' + message['content'] + '\\n\\n' }}{% elif message['role'] == 'assistant' %}{{ 'Assistant: ' + message['content'] + eos_token }}{% elif message['role'] == 'system' %}{{ message['content'] + '\\n\\n' }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ 'Assistant:' }}{% endif %}", + input: ChatTemplateInputs { + messages: example_chat.clone(), + add_generation_prompt: false, + bos_token: Some("<｜begin▁of▁sentence｜>"), + eos_token: Some("<｜end▁of▁sentence｜>"), + ..Default::default() + }, + target: "<｜begin▁of▁sentence｜>User: Hello, how are you?\n\nAssistant: I'm doing great. How can I help you today?<｜end▁of▁sentence｜>User: I'd like to show off how chat templating works!\n\n", + }, + ChatTemplateTestItem { + name: "h2oai/h2o-danube-1.8b-chat", + chat_template: "{% for message in messages %}{% if message['role'] == 'user' %}{{ '<|prompt|>' + message['content'] + eos_token }}{% elif message['role'] == 'system' %}{{ '<|system|>' + message['content'] + eos_token }}{% elif message['role'] == 'assistant' %}{{ '<|answer|>' + message['content'] + eos_token }}{% endif %}{% if loop.last and add_generation_prompt %}{{ '<|answer|>' }}{% endif %}{% endfor %}", + input: ChatTemplateInputs { + messages: example_chat.clone(), + add_generation_prompt: false, + bos_token: Some(""), + eos_token: Some(""), + ..Default::default() + }, + target: "<|prompt|>Hello, how are you?<|answer|>I'm doing great. How can I help you today?<|prompt|>I'd like to show off how chat templating works!", + }, + ChatTemplateTestItem { + name: "internlm/internlm2-chat-7b", + chat_template: "{% if messages[0]['role'] == 'user' or messages[0]['role'] == 'system' %}{{ bos_token }}{% endif %}{% for message in messages %}{{ '<|im_start|>' + message['role'] + '\\n' + message['content'] + '<|im_end|>' + '\\n' }}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\\n' }}{% elif messages[-1]['role'] == 'assistant' %}{{ eos_token }}{% endif %}", + input: ChatTemplateInputs { + messages: example_chat.clone(), + add_generation_prompt: false, + bos_token: Some(""), + eos_token: Some(""), + ..Default::default() + }, + target: "<|im_start|>user\nHello, how are you?<|im_end|>\n<|im_start|>assistant\nI'm doing great. How can I help you today?<|im_end|>\n<|im_start|>user\nI'd like to show off how chat templating works!<|im_end|>\n", + }, + ChatTemplateTestItem { + name: "TheBloke/deepseek-coder-33B-instruct-AWQ", + chat_template: "{%- set found_item = false -%}\n{%- for message in messages -%}\n {%- if message['role'] == 'system' -%}\n {%- set found_item = true -%}\n {%- endif -%}\n{%- endfor -%}\n{%- if not found_item -%}\n{{'You are an AI programming assistant, utilizing the Deepseek Coder model, developed by Deepseek Company, and you only answer questions related to computer science. For politically sensitive questions, security and privacy issues, and other non-computer science questions, you will refuse to answer.\\n'}}\n{%- endif %}\n{%- for message in messages %}\n {%- if message['role'] == 'system' %}\n{{ message['content'] }}\n {%- else %}\n {%- if message['role'] == 'user' %}\n{{'### Instruction:\\n' + message['content'] + '\\n'}}\n {%- else %}\n{{'### Response:\\n' + message['content'] + '\\n<|EOT|>\\n'}}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{{'### Response:\\n'}}\n", + input: ChatTemplateInputs { + messages: example_chat.clone(), + add_generation_prompt: false, + bos_token: Some("<｜begin▁of▁sentence｜>"), + eos_token: Some("<|EOT|>"), + ..Default::default() + }, + target: "You are an AI programming assistant, utilizing the Deepseek Coder model, developed by Deepseek Company, and you only answer questions related to computer science. For politically sensitive questions, security and privacy issues, and other non-computer science questions, you will refuse to answer.\n### Instruction:\nHello, how are you?\n### Response:\nI'm doing great. How can I help you today?\n<|EOT|>\n### Instruction:\nI'd like to show off how chat templating works!\n### Response:\n", + }, + ChatTemplateTestItem { + name: "ericzzz/falcon-rw-1b-chat", + // `.strip()` has been replaced with `| trim` in the following template + chat_template: "{% for message in messages %}{% if loop.index > 1 and loop.previtem['role'] != 'assistant' %}{{ ' ' }}{% endif %}{% if message['role'] == 'system' %}{{ '[SYS] ' + message['content'] | trim }}{% elif message['role'] == 'user' %}{{ '[INST] ' + message['content'] | trim }}{% elif message['role'] == 'assistant' %}{{ '[RESP] ' + message['content'] + eos_token }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ ' [RESP] ' }}{% endif %}", + input: ChatTemplateInputs { + messages: example_chat.clone(), + add_generation_prompt: false, + bos_token: Some("<|endoftext|>"), + eos_token: Some("<|endoftext|>"), + ..Default::default() + }, + target: "[INST] Hello, how are you? [RESP] I'm doing great. How can I help you today?<|endoftext|>[INST] I'd like to show off how chat templating works!", + }, + ChatTemplateTestItem { + name: "abacusai/Smaug-34B-v0.1", + chat_template: "{%- for idx in range(0, messages|length) -%}\n{%- if messages[idx]['role'] == 'user' -%}\n{%- if idx > 1 -%}\n{{- bos_token + '[INST] ' + messages[idx]['content'] + ' [/INST]' -}}\n{%- else -%}\n{{- messages[idx]['content'] + ' [/INST]' -}}\n{%- endif -%}\n{% elif messages[idx]['role'] == 'system' %}\n{{- '[INST] <>\\n' + messages[idx]['content'] + '\\n<>\\n\\n' -}}\n{%- elif messages[idx]['role'] == 'assistant' -%}\n{{- ' ' + messages[idx]['content'] + ' ' + eos_token -}}\n{% endif %}\n{% endfor %}", + input: ChatTemplateInputs { + messages: example_chat.clone(), + add_generation_prompt: false, + bos_token: Some(""), + eos_token: Some(""), + ..Default::default() + }, + target: "Hello, how are you? [/INST] I'm doing great. How can I help you today? [INST] I'd like to show off how chat templating works! [/INST]", + }, + ChatTemplateTestItem { + name: "maywell/Synatra-Mixtral-8x7B", + chat_template: "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n{% for message in messages %}{% if message['role'] == 'user' %}### Instruction:\n{{ message['content']|trim -}}{% if not loop.last %}{% endif %}\n{% elif message['role'] == 'assistant' %}### Response:\n{{ message['content']|trim -}}{% if not loop.last %}{% endif %}\n{% elif message['role'] == 'system' %}{{ message['content']|trim -}}{% if not loop.last %}{% endif %}\n{% endif %}\n{% endfor %}\n{% if add_generation_prompt and messages[-1]['role'] != 'assistant' %}\n### Response:\n{% endif %}", + input: ChatTemplateInputs { + messages: example_chat.clone(), + add_generation_prompt: false, + bos_token: Some(""), + eos_token: Some(""), + ..Default::default() + }, + target: "Below is an instruction that describes a task. Write a response that appropriately completes the request.### Instruction:Hello, how are you?### Response:I'm doing great. How can I help you today?### Instruction:I'd like to show off how chat templating works!", + }, + ChatTemplateTestItem { + name: "deepseek-ai/deepseek-coder-33b-instruct", + chat_template: "{% if not add_generation_prompt is defined %}\n{% set add_generation_prompt = false %}\n{% endif %}\n{%- set ns = namespace(found=false) -%}\n{%- for message in messages -%}\n {%- if message['role'] == 'system' -%}\n {%- set ns.found = true -%}\n {%- endif -%}\n{%- endfor -%}\n{{bos_token}}{%- if not ns.found -%}\n{{'You are an AI programming assistant, utilizing the Deepseek Coder model, developed by Deepseek Company, and you only answer questions related to computer science. For politically sensitive questions, security and privacy issues, and other non-computer science questions, you will refuse to answer\\n'}}\n{%- endif %}\n{%- for message in messages %}\n {%- if message['role'] == 'system' %}\n{{ message['content'] }}\n {%- else %}\n {%- if message['role'] == 'user' %}\n{{'### Instruction:\\n' + message['content'] + '\\n'}}\n {%- else %}\n{{'### Response:\\n' + message['content'] + '\\n<|EOT|>\\n'}}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{% if add_generation_prompt %}\n{{'### Response:'}}\n{% endif %}", + input: ChatTemplateInputs { + messages: example_chat.clone(), + add_generation_prompt: false, + bos_token: Some("<｜begin▁of▁sentence｜>"), + eos_token: Some(""), + ..Default::default() + }, + target: "<｜begin▁of▁sentence｜>You are an AI programming assistant, utilizing the Deepseek Coder model, developed by Deepseek Company, and you only answer questions related to computer science. For politically sensitive questions, security and privacy issues, and other non-computer science questions, you will refuse to answer\n### Instruction:\nHello, how are you?\n### Response:\nI'm doing great. How can I help you today?\n<|EOT|>\n### Instruction:\nI'd like to show off how chat templating works!\n", + }, + // NOT INCLUDED + // - meetkai/functionary-medium-v2.2 + // - fireworks-ai/firefunction-v1 + // https://github + ChatTemplateTestItem { + name: "maywell/PiVoT-MoE", + chat_template: "{{ (messages|selectattr('role', 'equalto', 'system')|list|last).content|trim if (messages|selectattr('role', 'equalto', 'system')|list) else '' }}{% for message in messages %}{% if message['role'] == 'system' %}{{ message['content']|trim }}{% elif message['role'] == 'user' %}### Instruction: {{ message['content']|trim }}{% elif message['role'] == 'assistant' %}### Response: {{ message['content']|trim }}{% elif message['role'] == 'user_context' %}### Input: {{ message['content']|trim }}{% endif %}{% if not loop.last %}\n{% endif %}{% endfor %}{% if add_generation_prompt and messages[-1]['role'] != 'assistant' %}### Response:{% endif %}", + input: ChatTemplateInputs { + messages: example_chat_with_system.clone(), + add_generation_prompt: false, + bos_token: Some(""), + eos_token: Some(""), + ..Default::default() + }, + target: "You are a friendly chatbot who always responds in the style of a pirateYou are a friendly chatbot who always responds in the style of a pirate### Instruction: Hello, how are you?### Response: I'm doing great. How can I help you today?### Instruction: I'd like to show off how chat templating works!", + }, + ]; -// #[allow(unused_variables)] // name is unused -// for ChatTemplateTestItem { -// name, -// chat_template, -// input, -// target, -// } in test_custom_templates -// { -// let mut env = Environment::new(); -// env.add_function("raise_exception", raise_exception); -// // trim all the whitespace -// let chat_template = chat_template -// .lines() -// .map(|line| line.trim()) -// .collect::>() -// .join(""); + #[allow(unused_variables)] // name is unused + for ChatTemplateTestItem { + name, + chat_template, + input, + target, + } in test_custom_templates + { + let mut env = Environment::new(); + env.add_function("raise_exception", raise_exception); + // trim all the whitespace + let chat_template = chat_template + .lines() + .map(|line| line.trim()) + .collect::>() + .join(""); -// let tmpl = env.template_from_str(&chat_template); -// let result = tmpl.unwrap().render(input).unwrap(); -// assert_eq!(result, target); -// } -// } -// } + let tmpl = env.template_from_str(&chat_template); + let result = tmpl.unwrap().render(input).unwrap(); + assert_eq!(result, target); + } + } +} diff --git a/router/src/lib.rs b/router/src/lib.rs index 3bdb77a7..3d2a2de4 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -525,7 +525,7 @@ impl ChatCompletion { pub(crate) fn new( model: String, system_fingerprint: String, - _output: Option, + output: Option, created: u64, details: Details, return_logprobs: bool, @@ -541,7 +541,7 @@ impl ChatCompletion { index: 0, message: Message { role: "assistant".into(), - content: None, + content: output, name: None, tool_calls, }, @@ -878,7 +878,7 @@ pub(crate) struct SerializedMessage { #[derive(Clone, Serialize, Deserialize, Default)] pub(crate) struct ChatTemplateInputs<'a> { - messages: Vec, + messages: Vec, bos_token: Option<&'a str>, eos_token: Option<&'a str>, add_generation_prompt: bool, @@ -914,13 +914,55 @@ pub(crate) struct Content { pub image_url: Option, } +mod message_content_serde { + use super::*; + use serde::de; + use serde::Deserializer; + use serde_json::Value; + + pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + let value = Value::deserialize(deserializer)?; + match value { + Value::String(s) => Ok(Some(s)), + Value::Array(arr) => { + let results: Result, _> = arr + .into_iter() + .map(|v| { + let content: Content = + serde_json::from_value(v).map_err(de::Error::custom)?; + match content.r#type.as_str() { + "text" => Ok(content.text.unwrap_or_default()), + "image_url" => { + if let Some(url) = content.image_url { + Ok(format!("\n![]({})", url.url)) + } else { + Ok(String::new()) + } + } + _ => Err(de::Error::custom("invalid content type")), + } + }) + .collect(); + + results.map(|strings| Some(strings.join(""))) + } + Value::Null => Ok(None), + _ => Err(de::Error::custom("invalid token format")), + } + } +} + #[derive(Clone, Deserialize, ToSchema, Serialize, Debug)] pub(crate) struct Message { #[schema(example = "user")] pub role: String, #[serde(skip_serializing_if = "Option::is_none")] #[schema(example = "My name is David and I")] - pub content: Option>, + #[serde(deserialize_with = "message_content_serde::deserialize")] + pub content: Option, #[serde(default, skip_serializing_if = "Option::is_none")] #[schema(example = "\"David\"")] pub name: Option,