mirror of
https://github.com/huggingface/text-generation-inference.git
synced 2025-05-31 19:22:07 +00:00
120d5773e8
* wip
wip
refacto
refacto
Initial setup for CXX binding to TRTLLM
Working FFI call for TGI and TRTLLM backend
Remove unused parameters annd force tokenizer name to be set
Overall build TRTLLM and deps through CMake build system
Enable end to end CMake build
First version loading engines and making it ready for inference
Remembering to check how we can detect support for chunked context
Move to latest TensorRT-LLM version
Specify which default log level to use depending on CMake build type
make leader executor mode working
unconditionally call InitializeBackend on the FFI layer
bind to CUDA::nvml to retrieve compute capabilities at runtime
updated logic and comment to detect cuda compute capabilities
implement the Stream method to send new tokens through a callback
use spdlog release 1.14.1 moving forward
update trtllm to latest version a96cccafcf6365c128f004f779160951f8c0801c
correctly tell cmake to build dependent tensorrt-llm required libraries
create cmake install target to put everything relevant in installation folder
add auth_token CLI argument to provide hf hub authentification token
allow converting huggingface::tokenizers error to TensorRtLlmBackendError
use correct include for spdlog
include guard to build example in cmakelists
working setup of the ffi layer
remove fmt import
use external fmt lib
end to end ffi flow working
make sure to track include/ffi.h to trigger rebuild from cargo
impl the rust backend which currently cannot move the actual computation in background thread
expose shutdown function at ffi layer
impl RwLock scenario for TensorRtLllmBackend
oops missing c++ backend definitions
compute the number of maximum new tokens for each request independently
make sure the context is not dropped in the middle of the async decoding.
remove unnecessary log
add all the necessary plumbery to return the generated content
update invalid doc in cpp file
correctly forward back the log probabilities
remove unneeded scope variable for now
refactor Stream impl for Generation to factorise code
expose the internal missing start/queue timestamp
forward tgi parameters rep/freq penalty
add some more validation about grammar not supported
define a shared struct to hold the result of a decoding step
expose information about potential error happening while decoding
remove logging
add logging in case of decoding error
make sure executor_worker is provided
add initial Dockerfile for TRTLLM backend
add some more information in CMakeLists.txt to correctly install executorWorker
add some more information in CMakeLists.txt to correctly find and install nvrtc wrapper
simplify prebuilt trtllm libraries name definition
do the same name definition stuff for tensorrt_llm_executor_static
leverage pkg-config to probe libraries paths and reuse new install structure from cmake
fix bad copy/past missing nvinfer linkage direction
align all the linker search dependency
add missing pkgconfig folder for MPI in Dockerfile
correctly setup linking search path for runtime layer
fix missing / before tgi lib path
adding missing ld_library_path for cuda stubs in Dockerfile
update tgi entrypoint
commenting out Python part for TensorRT installation
refactored docker image
move to TensorRT-LLM v0.11.0
make docker linter happy with same capitalization rule
fix typo
refactor the compute capabilities detection along with num gpus
update TensorRT-LLM to latest version
update TensorRT install script to latest
update build.rs to link to cuda 12.5
add missing dependant libraries for linking
clean up a bit
install to decoder_attention target
add some custom stuff for nccl linkage
fix envvar CARGO_CFG_TARGET_ARCH set at runtime vs compile time
use std::env::const::ARCH
make sure variable live long enough...
look for cuda 12.5
add some more basic info in README.md
* Rebase.
* Fix autodocs.
* Let's try to enable trtllm backend.
* Ignore backends/v3 by default.
* Fixing client.
* Fix makefile + autodocs.
* Updating the schema thing + redocly.
* Fix trtllm lint.
* Adding pb files ?
* Remove cargo fmt temporarily.
* ?
* Tmp.
* Remove both check + clippy ?
* Backporting telemetry.
* Backporting 457fb0a1
* Remove PB from git.
* Fixing PB with default member backends/client
* update TensorRT-LLM to latest version
* provided None for api_key
* link against libtensorrt_llm and not libtensorrt-llm
---------
Co-authored-by: OlivierDehaene <23298448+OlivierDehaene@users.noreply.github.com>
Co-authored-by: Morgan Funtowicz <morgan@huggingface.co>
759 lines
44 KiB
Rust
759 lines
44 KiB
Rust
use crate::infer::InferError;
|
||
use crate::{
|
||
ChatTemplateInputs, GrammarType, Message, MessageChunk, TextMessage, TokenizerConfigToken,
|
||
};
|
||
use minijinja::{Environment, ErrorKind, Template};
|
||
use minijinja_contrib::pycompat;
|
||
|
||
/// Raise a exception (custom function) used in the chat templates
|
||
pub(crate) fn raise_exception(err_text: String) -> Result<String, minijinja::Error> {
|
||
Err(minijinja::Error::new(ErrorKind::SyntaxError, err_text))
|
||
}
|
||
|
||
#[derive(Clone)]
|
||
pub(crate) struct ChatTemplate {
|
||
template: Template<'static, 'static>,
|
||
bos_token: Option<String>,
|
||
eos_token: Option<String>,
|
||
use_default_tool_template: bool,
|
||
}
|
||
|
||
impl ChatTemplate {
|
||
pub(crate) fn new(
|
||
template: String,
|
||
bos_token: Option<TokenizerConfigToken>,
|
||
eos_token: Option<TokenizerConfigToken>,
|
||
) -> Self {
|
||
let mut env = Box::new(Environment::new());
|
||
// enable things like .strip() or .capitalize()
|
||
env.set_unknown_method_callback(pycompat::unknown_method_callback);
|
||
let template_str = template.into_boxed_str();
|
||
env.add_function("raise_exception", raise_exception);
|
||
|
||
// check if contains the tools variable within the template
|
||
let use_default_tool_template =
|
||
!template_str.as_ref().replace(' ', "").contains("{{tools}}");
|
||
// leaking env and template_str as read-only, static resources for performance.
|
||
let template = Box::leak(env)
|
||
.template_from_str(Box::leak(template_str))
|
||
.unwrap();
|
||
|
||
Self {
|
||
template,
|
||
bos_token: bos_token.map(|token| token.as_str().to_string()),
|
||
eos_token: eos_token.map(|token| token.as_str().to_string()),
|
||
use_default_tool_template,
|
||
}
|
||
}
|
||
|
||
pub(crate) fn apply(
|
||
&self,
|
||
mut messages: Vec<Message>,
|
||
grammar_with_prompt: Option<(GrammarType, String)>,
|
||
) -> Result<String, InferError> {
|
||
if self.use_default_tool_template {
|
||
if let Some(last_message) = messages.last_mut() {
|
||
if let Some((GrammarType::Json(tools), tool_prompt)) = grammar_with_prompt {
|
||
last_message.content.push(MessageChunk::Text {
|
||
text: format!("\n---\n{}\n{}", tool_prompt, tools),
|
||
});
|
||
}
|
||
}
|
||
}
|
||
|
||
let messages: Vec<TextMessage> = messages.into_iter().map(|c| c.into()).collect();
|
||
|
||
self.template
|
||
.render(ChatTemplateInputs {
|
||
messages,
|
||
bos_token: self.bos_token.as_deref(),
|
||
eos_token: self.eos_token.as_deref(),
|
||
add_generation_prompt: true,
|
||
tools: None,
|
||
tools_prompt: None,
|
||
})
|
||
.map_err(InferError::TemplateError)
|
||
}
|
||
}
|
||
|
||
// tests
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use crate::infer::chat_template::raise_exception;
|
||
use crate::{ChatTemplateInputs, TextMessage};
|
||
use minijinja::Environment;
|
||
|
||
#[test]
|
||
fn test_chat_template() {
|
||
let env = Environment::new();
|
||
|
||
let source = r#"
|
||
{% for message in messages %}
|
||
{% if message['role'] == 'system' %}
|
||
{% if message['content']%}
|
||
{{'### System:\n' + message['content']+'\n\n'}}
|
||
{% endif %}
|
||
{% elif message['role'] == 'user' %}
|
||
{{'### User:\n' + message['content']+'\n\n'}}
|
||
{% elif message['role'] == 'assistant' %}
|
||
{{'### Assistant:\n' + message['content']}}
|
||
{% endif %}
|
||
{% if loop.last and add_generation_prompt %}
|
||
{{ '### Assistant:\n' }}
|
||
{% endif %}
|
||
{% endfor %}"#;
|
||
|
||
// trim all the whitespace
|
||
let source = source
|
||
.lines()
|
||
.map(|line| line.trim())
|
||
.collect::<Vec<&str>>()
|
||
.join("");
|
||
|
||
let tmpl = env.template_from_str(&source);
|
||
|
||
let chat_template_inputs = ChatTemplateInputs {
|
||
messages: vec![
|
||
TextMessage {
|
||
role: "user".to_string(),
|
||
content: "Hi!".to_string(),
|
||
},
|
||
TextMessage {
|
||
role: "assistant".to_string(),
|
||
content: "Hello how can I help?".to_string(),
|
||
},
|
||
TextMessage {
|
||
role: "user".to_string(),
|
||
content: "What is Deep Learning?".to_string(),
|
||
},
|
||
TextMessage {
|
||
role: "assistant".to_string(),
|
||
content: "magic!".to_string(),
|
||
},
|
||
],
|
||
bos_token: Some("[BOS]"),
|
||
eos_token: Some("[EOS]"),
|
||
add_generation_prompt: true,
|
||
..Default::default()
|
||
};
|
||
|
||
let result = tmpl.unwrap().render(chat_template_inputs).unwrap();
|
||
|
||
assert_eq!(
|
||
result,
|
||
"### User:\nHi!\n\n### Assistant:\nHello how can I help?### User:\nWhat is Deep Learning?\n\n### Assistant:\nmagic!### Assistant:\n"
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn test_chat_template_invalid_with_raise() {
|
||
let mut env = Environment::new();
|
||
env.add_function("raise_exception", raise_exception);
|
||
|
||
let source = r#"
|
||
{{ bos_token }}
|
||
{% for message in messages %}
|
||
{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}
|
||
{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}
|
||
{% endif %}
|
||
{% if message['role'] == 'user' %}
|
||
{{ '[INST] ' + message['content'] + ' [/INST]' }}
|
||
{% elif message['role'] == 'assistant' %}
|
||
{{ message['content'] + eos_token}}
|
||
{% else %}
|
||
{{ raise_exception('Only user and assistant roles are supported!') }}
|
||
{% endif %}
|
||
{% endfor %}"#;
|
||
|
||
// trim all the whitespace
|
||
let source = source
|
||
.lines()
|
||
.map(|line| line.trim())
|
||
.collect::<Vec<&str>>()
|
||
.join("");
|
||
|
||
let tmpl = env.template_from_str(&source);
|
||
|
||
let chat_template_inputs = ChatTemplateInputs {
|
||
messages: vec![
|
||
TextMessage {
|
||
role: "user".to_string(),
|
||
content: "Hi!".to_string(),
|
||
},
|
||
TextMessage {
|
||
role: "user".to_string(),
|
||
content: "Hi again!".to_string(),
|
||
},
|
||
TextMessage {
|
||
role: "assistant".to_string(),
|
||
content: "Hello how can I help?".to_string(),
|
||
},
|
||
TextMessage {
|
||
role: "user".to_string(),
|
||
content: "What is Deep Learning?".to_string(),
|
||
},
|
||
TextMessage {
|
||
role: "assistant".to_string(),
|
||
content: "magic!".to_string(),
|
||
},
|
||
],
|
||
bos_token: Some("[BOS]"),
|
||
eos_token: Some("[EOS]"),
|
||
add_generation_prompt: true,
|
||
..Default::default()
|
||
};
|
||
|
||
let result = tmpl.unwrap().render(chat_template_inputs); //.err().unwrap();
|
||
|
||
match result {
|
||
Ok(_) => panic!("Should have failed"),
|
||
Err(e) => {
|
||
assert_eq!(
|
||
e.detail().unwrap(),
|
||
"Conversation roles must alternate user/assistant/user/assistant/..."
|
||
);
|
||
}
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn test_chat_template_valid_with_raise() {
|
||
let mut env = Environment::new();
|
||
env.add_function("raise_exception", raise_exception);
|
||
|
||
let source = r#"
|
||
{{ bos_token }}
|
||
{% for message in messages %}
|
||
{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}
|
||
{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}
|
||
{% endif %}
|
||
{% if message['role'] == 'user' %}
|
||
{{ '[INST] ' + message['content'] + ' [/INST]' }}
|
||
{% elif message['role'] == 'assistant' %}
|
||
{{ message['content'] + eos_token}}
|
||
{% else %}
|
||
{{ raise_exception('Only user and assistant roles are supported!') }}
|
||
{% endif %}
|
||
{% endfor %}"#;
|
||
|
||
// trim all the whitespace
|
||
let source = source
|
||
.lines()
|
||
.map(|line| line.trim())
|
||
.collect::<Vec<&str>>()
|
||
.join("");
|
||
|
||
let tmpl = env.template_from_str(&source);
|
||
|
||
let chat_template_inputs = ChatTemplateInputs {
|
||
messages: vec![
|
||
TextMessage {
|
||
role: "user".to_string(),
|
||
content: "Hi!".to_string(),
|
||
},
|
||
TextMessage {
|
||
role: "assistant".to_string(),
|
||
content: "Hello how can I help?".to_string(),
|
||
},
|
||
TextMessage {
|
||
role: "user".to_string(),
|
||
content: "What is Deep Learning?".to_string(),
|
||
},
|
||
TextMessage {
|
||
role: "assistant".to_string(),
|
||
content: "magic!".to_string(),
|
||
},
|
||
],
|
||
bos_token: Some("[BOS]"),
|
||
eos_token: Some("[EOS]"),
|
||
add_generation_prompt: true,
|
||
..Default::default()
|
||
};
|
||
|
||
let result = tmpl.unwrap().render(chat_template_inputs).unwrap();
|
||
assert_eq!(result, "[BOS][INST] Hi! [/INST]Hello how can I help?[EOS][INST] What is Deep Learning? [/INST]magic![EOS]");
|
||
}
|
||
|
||
#[test]
|
||
fn test_chat_template_valid_with_add_generation_prompt() {
|
||
let mut env = Environment::new();
|
||
env.add_function("raise_exception", raise_exception);
|
||
|
||
let source = r#"
|
||
{% for message in messages %}
|
||
{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}
|
||
{% endfor %}
|
||
{% if add_generation_prompt %}
|
||
{{ '<|im_start|>assistant\n' }}
|
||
{% endif %}"#;
|
||
|
||
// trim all the whitespace
|
||
let source = source
|
||
.lines()
|
||
.map(|line| line.trim())
|
||
.collect::<Vec<&str>>()
|
||
.join("");
|
||
|
||
let tmpl = env.template_from_str(&source);
|
||
|
||
let chat_template_inputs = ChatTemplateInputs {
|
||
messages: vec![
|
||
TextMessage {
|
||
role: "user".to_string(),
|
||
content: "Hi!".to_string(),
|
||
},
|
||
TextMessage {
|
||
role: "assistant".to_string(),
|
||
content: "Hello how can I help?".to_string(),
|
||
},
|
||
TextMessage {
|
||
role: "user".to_string(),
|
||
content: "What is Deep Learning?".to_string(),
|
||
},
|
||
TextMessage {
|
||
role: "assistant".to_string(),
|
||
content: "magic!".to_string(),
|
||
},
|
||
],
|
||
bos_token: Some("[BOS]"),
|
||
eos_token: Some("[EOS]"),
|
||
add_generation_prompt: true,
|
||
..Default::default()
|
||
};
|
||
|
||
let result = tmpl.unwrap().render(chat_template_inputs).unwrap();
|
||
assert_eq!(result, "<|im_start|>user\nHi!<|im_end|>\n<|im_start|>assistant\nHello how can I help?<|im_end|>\n<|im_start|>user\nWhat is Deep Learning?<|im_end|>\n<|im_start|>assistant\nmagic!<|im_end|>\n<|im_start|>assistant\n");
|
||
}
|
||
|
||
struct ChatTemplateTestItem {
|
||
name: &'static str,
|
||
chat_template: &'static str,
|
||
input: ChatTemplateInputs<'static>,
|
||
target: &'static str,
|
||
}
|
||
|
||
#[test]
|
||
fn test_many_chat_templates() {
|
||
let example_chat = vec![
|
||
TextMessage {
|
||
role: "user".to_string(),
|
||
content: "Hello, how are you?".to_string(),
|
||
},
|
||
TextMessage {
|
||
role: "assistant".to_string(),
|
||
content: "I'm doing great. How can I help you today?".to_string(),
|
||
},
|
||
TextMessage {
|
||
role: "user".to_string(),
|
||
content: "I'd like to show off how chat templating works!".to_string(),
|
||
},
|
||
];
|
||
|
||
let example_chat_with_system = [TextMessage {
|
||
role: "system".to_string(),
|
||
content: "You are a friendly chatbot who always responds in the style of a pirate"
|
||
.to_string(),
|
||
}]
|
||
.iter()
|
||
.chain(&example_chat)
|
||
.cloned()
|
||
.collect::<Vec<_>>();
|
||
|
||
let test_default_templates = vec![
|
||
ChatTemplateTestItem {
|
||
name: "_base",
|
||
chat_template: "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\\n' + message['content'] + '<|im_end|>' + '\\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\\n' }}{% endif %}",
|
||
input: ChatTemplateInputs {
|
||
messages: example_chat.clone(),
|
||
add_generation_prompt: false,
|
||
bos_token: Some(""),
|
||
eos_token: Some(""),
|
||
..Default::default()
|
||
},
|
||
target: "<|im_start|>user\nHello, how are you?<|im_end|>\n<|im_start|>assistant\nI'm doing great. How can I help you today?<|im_end|>\n<|im_start|>user\nI'd like to show off how chat templating works!<|im_end|>\n",
|
||
},
|
||
ChatTemplateTestItem {
|
||
name: "blenderbot",
|
||
chat_template: "{% for message in messages %}{% if message['role'] == 'user' %}{{ ' ' }}{% endif %}{{ message['content'] }}{% if not loop.last %}{{ ' ' }}{% endif %}{% endfor %}{{ eos_token }}",
|
||
input: ChatTemplateInputs {
|
||
messages: example_chat.clone(),
|
||
add_generation_prompt: false,
|
||
bos_token: Some(""),
|
||
eos_token: Some("</s>"),
|
||
..Default::default()
|
||
},
|
||
target: " Hello, how are you? I'm doing great. How can I help you today? I'd like to show off how chat templating works!</s>",
|
||
},
|
||
ChatTemplateTestItem {
|
||
name: "blenderbot_small",
|
||
chat_template: "{% for message in messages %}{% if message['role'] == 'user' %}{{ ' ' }}{% endif %}{{ message['content'] }}{% if not loop.last %}{{ ' ' }}{% endif %}{% endfor %}{{ eos_token }}",
|
||
input: ChatTemplateInputs {
|
||
messages: example_chat.clone(),
|
||
add_generation_prompt: false,
|
||
bos_token: Some(""),
|
||
eos_token: Some("</s>"),
|
||
..Default::default()
|
||
},
|
||
target: " Hello, how are you? I'm doing great. How can I help you today? I'd like to show off how chat templating works!</s>",
|
||
},
|
||
ChatTemplateTestItem {
|
||
name: "bloom",
|
||
chat_template: "{% for message in messages %}{{ message.content }}{{ eos_token }}{% endfor %}",
|
||
input: ChatTemplateInputs {
|
||
messages: example_chat.clone(),
|
||
add_generation_prompt: false,
|
||
bos_token: Some(""),
|
||
eos_token: Some("</s>"),
|
||
..Default::default()
|
||
},
|
||
target: "Hello, how are you?</s>I'm doing great. How can I help you today?</s>I'd like to show off how chat templating works!</s>",
|
||
},
|
||
ChatTemplateTestItem {
|
||
name: "gpt_neox",
|
||
chat_template: "{% for message in messages %}{{ message.content }}{{ eos_token }}{% endfor %}",
|
||
input: ChatTemplateInputs {
|
||
messages: example_chat.clone(),
|
||
add_generation_prompt: false,
|
||
bos_token: Some(""),
|
||
eos_token: Some("<|endoftext|>"),
|
||
..Default::default()
|
||
},
|
||
target: "Hello, how are you?<|endoftext|>I'm doing great. How can I help you today?<|endoftext|>I'd like to show off how chat templating works!<|endoftext|>",
|
||
},
|
||
ChatTemplateTestItem {
|
||
name: "gpt2",
|
||
chat_template: "{% for message in messages %}{{ message.content }}{{ eos_token }}{% endfor %}",
|
||
input: ChatTemplateInputs {
|
||
messages: example_chat.clone(),
|
||
add_generation_prompt: false,
|
||
bos_token: Some(""),
|
||
eos_token: Some("<|endoftext|>"),
|
||
..Default::default()
|
||
},
|
||
target: "Hello, how are you?<|endoftext|>I'm doing great. How can I help you today?<|endoftext|>I'd like to show off how chat templating works!<|endoftext|>",
|
||
},
|
||
ChatTemplateTestItem {
|
||
name: "llama",
|
||
// NOTE: the `.strip()` has been replaced with `| trim` in the following template
|
||
chat_template: "{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% elif USE_DEFAULT_PROMPT == true and not '<<SYS>>' in messages[0]['content'] %}{% set loop_messages = messages %}{% set system_message = 'DEFAULT_SYSTEM_MESSAGE' %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 and system_message != false %}{% set content = '<<SYS>>\\n' + system_message + '\\n<</SYS>>\\n\\n' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ bos_token +'[INST] ' + content | trim + ' [/INST]' }}{% elif message['role'] == 'system' %}{{ '<<SYS>>\\n' + content | trim + '\\n<</SYS>>\\n\\n' }}{% elif message['role'] == 'assistant' %}{{ ' ' + content | trim + ' ' + eos_token }}{% endif %}{% endfor %}",
|
||
input: ChatTemplateInputs {
|
||
messages: example_chat_with_system.clone(),
|
||
add_generation_prompt: true,
|
||
bos_token: Some("<s>"),
|
||
eos_token: Some("</s>"),
|
||
..Default::default()
|
||
},
|
||
target: "<s>[INST] <<SYS>>\nYou are a friendly chatbot who always responds in the style of a pirate\n<</SYS>>\n\nHello, how are you? [/INST] I'm doing great. How can I help you today? </s><s>[INST] I'd like to show off how chat templating works! [/INST]",
|
||
},
|
||
ChatTemplateTestItem {
|
||
name: "whisper",
|
||
chat_template: "{% for message in messages %}{{ message.content }}{{ eos_token }}{% endfor %}",
|
||
input: ChatTemplateInputs {
|
||
messages: example_chat.clone(),
|
||
add_generation_prompt: true,
|
||
bos_token: Some(""),
|
||
eos_token: Some("<|endoftext|>"),
|
||
..Default::default()
|
||
},
|
||
target: "Hello, how are you?<|endoftext|>I'm doing great. How can I help you today?<|endoftext|>I'd like to show off how chat templating works!<|endoftext|>",
|
||
},
|
||
];
|
||
|
||
#[allow(unused_variables)] // name is unused
|
||
for ChatTemplateTestItem {
|
||
name,
|
||
chat_template,
|
||
input,
|
||
target,
|
||
} in test_default_templates
|
||
{
|
||
let mut env = Environment::new();
|
||
env.add_function("raise_exception", raise_exception);
|
||
let tmpl = env.template_from_str(chat_template);
|
||
let result = tmpl.unwrap().render(input).unwrap();
|
||
assert_eq!(result, target);
|
||
}
|
||
|
||
let test_custom_templates = vec![
|
||
ChatTemplateTestItem {
|
||
name: "HuggingFaceH4/zephyr-7b-beta (add_generation_prompt=false)",
|
||
chat_template: "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\\n' + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
|
||
input: ChatTemplateInputs {
|
||
messages: example_chat_with_system.clone(),
|
||
add_generation_prompt: false,
|
||
bos_token: Some(""),
|
||
eos_token: Some("</s>"),
|
||
..Default::default()
|
||
},
|
||
target: "<|system|>\nYou are a friendly chatbot who always responds in the style of a pirate</s><|user|>\nHello, how are you?</s><|assistant|>\nI'm doing great. How can I help you today?</s><|user|>\nI'd like to show off how chat templating works!</s>",
|
||
},
|
||
ChatTemplateTestItem {
|
||
name: "HuggingFaceH4/zephyr-7b-beta (add_generation_prompt=true)",
|
||
chat_template: "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\\n' + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
|
||
input: ChatTemplateInputs {
|
||
messages: vec![
|
||
TextMessage {
|
||
role: "system".to_string(),
|
||
content: "You are a friendly chatbot who always responds in the style of a pirate".to_string(),
|
||
},
|
||
TextMessage {
|
||
role: "user".to_string(),
|
||
content: "How many helicopters can a human eat in one sitting?".to_string(),
|
||
},
|
||
],
|
||
add_generation_prompt: true,
|
||
bos_token: Some(""),
|
||
eos_token: Some("</s>"),
|
||
..Default::default()
|
||
},
|
||
target: "<|system|>\nYou are a friendly chatbot who always responds in the style of a pirate</s><|user|>\nHow many helicopters can a human eat in one sitting?</s><|assistant|>",
|
||
},
|
||
ChatTemplateTestItem {
|
||
name: "HuggingFaceH4/zephyr-7b-gemma-v0.1",
|
||
chat_template: "{% if messages[0]['role'] == 'user' or messages[0]['role'] == 'system' %}{{ bos_token }}{% endif %}{% for message in messages %}{{ '<|im_start|>' + message['role'] + '\\n' + message['content'] + '<|im_end|>' + '\\n' }}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% elif messages[-1]['role'] == 'assistant' %}{{ eos_token }}{% endif %}",
|
||
input: ChatTemplateInputs {
|
||
messages: example_chat.clone(),
|
||
add_generation_prompt: false,
|
||
bos_token: Some("<bos>"),
|
||
eos_token: Some("<eos>"),
|
||
..Default::default()
|
||
},
|
||
target: "<bos><|im_start|>user\nHello, how are you?<|im_end|>\n<|im_start|>assistant\nI'm doing great. How can I help you today?<|im_end|>\n<|im_start|>user\nI'd like to show off how chat templating works!<|im_end|>\n",
|
||
},
|
||
ChatTemplateTestItem {
|
||
name: "mistralai/Mistral-7B-Instruct-v0.1",
|
||
chat_template: "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token + ' ' }}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %}",
|
||
input: ChatTemplateInputs {
|
||
messages: example_chat.clone(),
|
||
add_generation_prompt: false,
|
||
bos_token: Some("<s>"),
|
||
eos_token: Some("</s>"),
|
||
..Default::default()
|
||
},
|
||
target: "<s>[INST] Hello, how are you? [/INST]I'm doing great. How can I help you today?</s> [INST] I'd like to show off how chat templating works! [/INST]",
|
||
},
|
||
ChatTemplateTestItem {
|
||
name: "mistralai/Mixtral-8x7B-Instruct-v0.1",
|
||
chat_template: "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token}}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %}",
|
||
input: ChatTemplateInputs {
|
||
messages: example_chat.clone(),
|
||
add_generation_prompt: false,
|
||
bos_token: Some("<s>"),
|
||
eos_token: Some("</s>"),
|
||
..Default::default()
|
||
},
|
||
target: "<s>[INST] Hello, how are you? [/INST]I'm doing great. How can I help you today?</s>[INST] I'd like to show off how chat templating works! [/INST]",
|
||
},
|
||
ChatTemplateTestItem {
|
||
name: "cognitivecomputations/dolphin-2.5-mixtral-8x7b",
|
||
chat_template: "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\\n' + message['content'] + '<|im_end|>' + '\\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\\n' }}{% endif %}",
|
||
input: ChatTemplateInputs {
|
||
messages: example_chat.clone(),
|
||
add_generation_prompt: false,
|
||
bos_token: Some("<s>"),
|
||
eos_token: Some("</s>"),
|
||
..Default::default()
|
||
},
|
||
target: "<|im_start|>user\nHello, how are you?<|im_end|>\n<|im_start|>assistant\nI'm doing great. How can I help you today?<|im_end|>\n<|im_start|>user\nI'd like to show off how chat templating works!<|im_end|>\n",
|
||
},
|
||
ChatTemplateTestItem {
|
||
name: "openchat/openchat-3.5-0106",
|
||
// `.title()` has been replaced with `| upper` in the following template
|
||
chat_template: "{{ bos_token }}{% for message in messages %}{{ 'GPT4 Correct ' + (message['role'] | title) + ': ' + message['content'] + '<|end_of_turn|>'}}{% endfor %}{% if add_generation_prompt %}{{ 'GPT4 Correct Assistant:' }}{% endif %}",
|
||
input: ChatTemplateInputs {
|
||
messages: example_chat.clone(),
|
||
add_generation_prompt: false,
|
||
bos_token: Some("<s>"),
|
||
eos_token: Some("</s>"),
|
||
..Default::default()
|
||
},
|
||
target: "<s>GPT4 Correct User: Hello, how are you?<|end_of_turn|>GPT4 Correct Assistant: I'm doing great. How can I help you today?<|end_of_turn|>GPT4 Correct User: I'd like to show off how chat templating works!<|end_of_turn|>",
|
||
},
|
||
ChatTemplateTestItem {
|
||
name: "upstage/SOLAR-10.7B-Instruct-v1.0",
|
||
chat_template: "{% for message in messages %}{{ message.content }}{{ eos_token }}{% endfor %}",
|
||
input: ChatTemplateInputs {
|
||
messages: example_chat.clone(),
|
||
add_generation_prompt: false,
|
||
bos_token: Some("<s>"),
|
||
eos_token: Some("</s>"),
|
||
..Default::default()
|
||
},
|
||
target: "Hello, how are you?</s>I'm doing great. How can I help you today?</s>I'd like to show off how chat templating works!</s>",
|
||
},
|
||
ChatTemplateTestItem {
|
||
name: "codellama/CodeLlama-70b-Instruct-hf",
|
||
// NOTE: `.strip()` has been replaced with `| trim` in the following template
|
||
chat_template: "{% if messages[0]['role'] == 'system' %}{% set user_index = 1 %}{% else %}{% set user_index = 0 %}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != ((loop.index0 + user_index) % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 %}{{ '<s>' }}{% endif %}{% set content = 'Source: ' + message['role'] + '\\n\\n ' + message['content'] | trim %}{{ content + ' <step> ' }}{% endfor %}{{'Source: assistant\\nDestination: user\\n\\n '}}",
|
||
input: ChatTemplateInputs {
|
||
messages: example_chat.clone(),
|
||
add_generation_prompt: false,
|
||
bos_token: Some("<s>"),
|
||
eos_token: Some("</s>"),
|
||
..Default::default()
|
||
},
|
||
target: "<s>Source: user\n\n Hello, how are you? <step> Source: assistant\n\n I'm doing great. How can I help you today? <step> Source: user\n\n I'd like to show off how chat templating works! <step> Source: assistant\nDestination: user\n\n ",
|
||
},
|
||
ChatTemplateTestItem {
|
||
name: "Deci/DeciLM-7B-instruct",
|
||
chat_template: "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '### User:\\n' + message['content'] }}\n{% elif message['role'] == 'system' %}\n{{ '### System:\\n' + message['content'] }}\n{% elif message['role'] == 'assistant' %}\n{{ '### Assistant:\\n' + message['content'] }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '### Assistant:' }}\n{% endif %}\n{% endfor %}",
|
||
input: ChatTemplateInputs {
|
||
messages: example_chat.clone(),
|
||
add_generation_prompt: false,
|
||
bos_token: Some("<s>"),
|
||
eos_token: Some("</s>"),
|
||
..Default::default()
|
||
},
|
||
target: "### User:\nHello, how are you?### Assistant:\nI'm doing great. How can I help you today?### User:\nI'd like to show off how chat templating works!",
|
||
},
|
||
ChatTemplateTestItem {
|
||
name: "Qwen/Qwen1.5-72B-Chat",
|
||
chat_template: "{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system\\nYou are a helpful assistant<|im_end|>\\n' }}{% endif %}{{'<|im_start|>' + message['role'] + '\\n' + message['content']}}{% if (loop.last and add_generation_prompt) or not loop.last %}{{ '<|im_end|>' + '\\n'}}{% endif %}{% endfor %}{% if add_generation_prompt and messages[-1]['role'] != 'assistant' %}{{ '<|im_start|>assistant\\n' }}{% endif %}",
|
||
input: ChatTemplateInputs {
|
||
messages: example_chat.clone(),
|
||
add_generation_prompt: false,
|
||
bos_token: Some("<s>"),
|
||
eos_token: Some("</s>"),
|
||
..Default::default()
|
||
},
|
||
target: "<|im_start|>system\nYou are a helpful assistant<|im_end|>\n<|im_start|>user\nHello, how are you?<|im_end|>\n<|im_start|>assistant\nI'm doing great. How can I help you today?<|im_end|>\n<|im_start|>user\nI'd like to show off how chat templating works!",
|
||
},
|
||
ChatTemplateTestItem {
|
||
name: "deepseek-ai/deepseek-llm-7b-chat",
|
||
chat_template: "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{{ bos_token }}{% for message in messages %}{% if message['role'] == 'user' %}{{ 'User: ' + message['content'] + '\\n\\n' }}{% elif message['role'] == 'assistant' %}{{ 'Assistant: ' + message['content'] + eos_token }}{% elif message['role'] == 'system' %}{{ message['content'] + '\\n\\n' }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ 'Assistant:' }}{% endif %}",
|
||
input: ChatTemplateInputs {
|
||
messages: example_chat.clone(),
|
||
add_generation_prompt: false,
|
||
bos_token: Some("<｜begin▁of▁sentence｜>"),
|
||
eos_token: Some("<｜end▁of▁sentence｜>"),
|
||
..Default::default()
|
||
},
|
||
target: "<｜begin▁of▁sentence｜>User: Hello, how are you?\n\nAssistant: I'm doing great. How can I help you today?<｜end▁of▁sentence｜>User: I'd like to show off how chat templating works!\n\n",
|
||
},
|
||
ChatTemplateTestItem {
|
||
name: "h2oai/h2o-danube-1.8b-chat",
|
||
chat_template: "{% for message in messages %}{% if message['role'] == 'user' %}{{ '<|prompt|>' + message['content'] + eos_token }}{% elif message['role'] == 'system' %}{{ '<|system|>' + message['content'] + eos_token }}{% elif message['role'] == 'assistant' %}{{ '<|answer|>' + message['content'] + eos_token }}{% endif %}{% if loop.last and add_generation_prompt %}{{ '<|answer|>' }}{% endif %}{% endfor %}",
|
||
input: ChatTemplateInputs {
|
||
messages: example_chat.clone(),
|
||
add_generation_prompt: false,
|
||
bos_token: Some("<s>"),
|
||
eos_token: Some("</s>"),
|
||
..Default::default()
|
||
},
|
||
target: "<|prompt|>Hello, how are you?</s><|answer|>I'm doing great. How can I help you today?</s><|prompt|>I'd like to show off how chat templating works!</s>",
|
||
},
|
||
ChatTemplateTestItem {
|
||
name: "internlm/internlm2-chat-7b",
|
||
chat_template: "{% if messages[0]['role'] == 'user' or messages[0]['role'] == 'system' %}{{ bos_token }}{% endif %}{% for message in messages %}{{ '<|im_start|>' + message['role'] + '\\n' + message['content'] + '<|im_end|>' + '\\n' }}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\\n' }}{% elif messages[-1]['role'] == 'assistant' %}{{ eos_token }}{% endif %}",
|
||
input: ChatTemplateInputs {
|
||
messages: example_chat.clone(),
|
||
add_generation_prompt: false,
|
||
bos_token: Some("<s>"),
|
||
eos_token: Some("</s>"),
|
||
..Default::default()
|
||
},
|
||
target: "<s><|im_start|>user\nHello, how are you?<|im_end|>\n<|im_start|>assistant\nI'm doing great. How can I help you today?<|im_end|>\n<|im_start|>user\nI'd like to show off how chat templating works!<|im_end|>\n",
|
||
},
|
||
ChatTemplateTestItem {
|
||
name: "TheBloke/deepseek-coder-33B-instruct-AWQ",
|
||
chat_template: "{%- set found_item = false -%}\n{%- for message in messages -%}\n {%- if message['role'] == 'system' -%}\n {%- set found_item = true -%}\n {%- endif -%}\n{%- endfor -%}\n{%- if not found_item -%}\n{{'You are an AI programming assistant, utilizing the Deepseek Coder model, developed by Deepseek Company, and you only answer questions related to computer science. For politically sensitive questions, security and privacy issues, and other non-computer science questions, you will refuse to answer.\\n'}}\n{%- endif %}\n{%- for message in messages %}\n {%- if message['role'] == 'system' %}\n{{ message['content'] }}\n {%- else %}\n {%- if message['role'] == 'user' %}\n{{'### Instruction:\\n' + message['content'] + '\\n'}}\n {%- else %}\n{{'### Response:\\n' + message['content'] + '\\n<|EOT|>\\n'}}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{{'### Response:\\n'}}\n",
|
||
input: ChatTemplateInputs {
|
||
messages: example_chat.clone(),
|
||
add_generation_prompt: false,
|
||
bos_token: Some("<｜begin▁of▁sentence｜>"),
|
||
eos_token: Some("<|EOT|>"),
|
||
..Default::default()
|
||
},
|
||
target: "You are an AI programming assistant, utilizing the Deepseek Coder model, developed by Deepseek Company, and you only answer questions related to computer science. For politically sensitive questions, security and privacy issues, and other non-computer science questions, you will refuse to answer.\n### Instruction:\nHello, how are you?\n### Response:\nI'm doing great. How can I help you today?\n<|EOT|>\n### Instruction:\nI'd like to show off how chat templating works!\n### Response:\n",
|
||
},
|
||
ChatTemplateTestItem {
|
||
name: "ericzzz/falcon-rw-1b-chat",
|
||
// `.strip()` has been replaced with `| trim` in the following template
|
||
chat_template: "{% for message in messages %}{% if loop.index > 1 and loop.previtem['role'] != 'assistant' %}{{ ' ' }}{% endif %}{% if message['role'] == 'system' %}{{ '[SYS] ' + message['content'] | trim }}{% elif message['role'] == 'user' %}{{ '[INST] ' + message['content'] | trim }}{% elif message['role'] == 'assistant' %}{{ '[RESP] ' + message['content'] + eos_token }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ ' [RESP] ' }}{% endif %}",
|
||
input: ChatTemplateInputs {
|
||
messages: example_chat.clone(),
|
||
add_generation_prompt: false,
|
||
bos_token: Some("<|endoftext|>"),
|
||
eos_token: Some("<|endoftext|>"),
|
||
..Default::default()
|
||
},
|
||
target: "[INST] Hello, how are you? [RESP] I'm doing great. How can I help you today?<|endoftext|>[INST] I'd like to show off how chat templating works!",
|
||
},
|
||
ChatTemplateTestItem {
|
||
name: "abacusai/Smaug-34B-v0.1",
|
||
chat_template: "{%- for idx in range(0, messages|length) -%}\n{%- if messages[idx]['role'] == 'user' -%}\n{%- if idx > 1 -%}\n{{- bos_token + '[INST] ' + messages[idx]['content'] + ' [/INST]' -}}\n{%- else -%}\n{{- messages[idx]['content'] + ' [/INST]' -}}\n{%- endif -%}\n{% elif messages[idx]['role'] == 'system' %}\n{{- '[INST] <<SYS>>\\n' + messages[idx]['content'] + '\\n<</SYS>>\\n\\n' -}}\n{%- elif messages[idx]['role'] == 'assistant' -%}\n{{- ' ' + messages[idx]['content'] + ' ' + eos_token -}}\n{% endif %}\n{% endfor %}",
|
||
input: ChatTemplateInputs {
|
||
messages: example_chat.clone(),
|
||
add_generation_prompt: false,
|
||
bos_token: Some("<s>"),
|
||
eos_token: Some("</s>"),
|
||
..Default::default()
|
||
},
|
||
target: "Hello, how are you? [/INST] I'm doing great. How can I help you today? </s><s>[INST] I'd like to show off how chat templating works! [/INST]",
|
||
},
|
||
ChatTemplateTestItem {
|
||
name: "maywell/Synatra-Mixtral-8x7B",
|
||
chat_template: "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n{% for message in messages %}{% if message['role'] == 'user' %}### Instruction:\n{{ message['content']|trim -}}{% if not loop.last %}{% endif %}\n{% elif message['role'] == 'assistant' %}### Response:\n{{ message['content']|trim -}}{% if not loop.last %}{% endif %}\n{% elif message['role'] == 'system' %}{{ message['content']|trim -}}{% if not loop.last %}{% endif %}\n{% endif %}\n{% endfor %}\n{% if add_generation_prompt and messages[-1]['role'] != 'assistant' %}\n### Response:\n{% endif %}",
|
||
input: ChatTemplateInputs {
|
||
messages: example_chat.clone(),
|
||
add_generation_prompt: false,
|
||
bos_token: Some("<s>"),
|
||
eos_token: Some("</s>"),
|
||
..Default::default()
|
||
},
|
||
target: "Below is an instruction that describes a task. Write a response that appropriately completes the request.### Instruction:Hello, how are you?### Response:I'm doing great. How can I help you today?### Instruction:I'd like to show off how chat templating works!",
|
||
},
|
||
ChatTemplateTestItem {
|
||
name: "deepseek-ai/deepseek-coder-33b-instruct",
|
||
chat_template: "{% if not add_generation_prompt is defined %}\n{% set add_generation_prompt = false %}\n{% endif %}\n{%- set ns = namespace(found=false) -%}\n{%- for message in messages -%}\n {%- if message['role'] == 'system' -%}\n {%- set ns.found = true -%}\n {%- endif -%}\n{%- endfor -%}\n{{bos_token}}{%- if not ns.found -%}\n{{'You are an AI programming assistant, utilizing the Deepseek Coder model, developed by Deepseek Company, and you only answer questions related to computer science. For politically sensitive questions, security and privacy issues, and other non-computer science questions, you will refuse to answer\\n'}}\n{%- endif %}\n{%- for message in messages %}\n {%- if message['role'] == 'system' %}\n{{ message['content'] }}\n {%- else %}\n {%- if message['role'] == 'user' %}\n{{'### Instruction:\\n' + message['content'] + '\\n'}}\n {%- else %}\n{{'### Response:\\n' + message['content'] + '\\n<|EOT|>\\n'}}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{% if add_generation_prompt %}\n{{'### Response:'}}\n{% endif %}",
|
||
input: ChatTemplateInputs {
|
||
messages: example_chat.clone(),
|
||
add_generation_prompt: false,
|
||
bos_token: Some("<｜begin▁of▁sentence｜>"),
|
||
eos_token: Some("</EOT>"),
|
||
..Default::default()
|
||
},
|
||
target: "<｜begin▁of▁sentence｜>You are an AI programming assistant, utilizing the Deepseek Coder model, developed by Deepseek Company, and you only answer questions related to computer science. For politically sensitive questions, security and privacy issues, and other non-computer science questions, you will refuse to answer\n### Instruction:\nHello, how are you?\n### Response:\nI'm doing great. How can I help you today?\n<|EOT|>\n### Instruction:\nI'd like to show off how chat templating works!\n",
|
||
},
|
||
// NOT INCLUDED
|
||
// - meetkai/functionary-medium-v3.2
|
||
// - fireworks-ai/firefunction-v1
|
||
// https://github
|
||
ChatTemplateTestItem {
|
||
name: "maywell/PiVoT-MoE",
|
||
chat_template: "{{ (messages|selectattr('role', 'equalto', 'system')|list|last).content|trim if (messages|selectattr('role', 'equalto', 'system')|list) else '' }}{% for message in messages %}{% if message['role'] == 'system' %}{{ message['content']|trim }}{% elif message['role'] == 'user' %}### Instruction: {{ message['content']|trim }}{% elif message['role'] == 'assistant' %}### Response: {{ message['content']|trim }}{% elif message['role'] == 'user_context' %}### Input: {{ message['content']|trim }}{% endif %}{% if not loop.last %}\n{% endif %}{% endfor %}{% if add_generation_prompt and messages[-1]['role'] != 'assistant' %}### Response:{% endif %}",
|
||
input: ChatTemplateInputs {
|
||
messages: example_chat_with_system.clone(),
|
||
add_generation_prompt: false,
|
||
bos_token: Some("<s>"),
|
||
eos_token: Some("</s>"),
|
||
..Default::default()
|
||
},
|
||
target: "You are a friendly chatbot who always responds in the style of a pirateYou are a friendly chatbot who always responds in the style of a pirate### Instruction: Hello, how are you?### Response: I'm doing great. How can I help you today?### Instruction: I'd like to show off how chat templating works!",
|
||
},
|
||
];
|
||
|
||
#[allow(unused_variables)] // name is unused
|
||
for ChatTemplateTestItem {
|
||
name,
|
||
chat_template,
|
||
input,
|
||
target,
|
||
} in test_custom_templates
|
||
{
|
||
let mut env = Environment::new();
|
||
env.add_function("raise_exception", raise_exception);
|
||
// trim all the whitespace
|
||
let chat_template = chat_template
|
||
.lines()
|
||
.map(|line| line.trim())
|
||
.collect::<Vec<&str>>()
|
||
.join("");
|
||
|
||
let tmpl = env.template_from_str(&chat_template);
|
||
let result = tmpl.unwrap().render(input).unwrap();
|
||
assert_eq!(result, target);
|
||
}
|
||
}
|
||
}
|