huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-07-26 18:00:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

add some more validation about grammar not supported

Browse Source
This commit is contained in:
Morgan Funtowicz 2024-07-18 20:57:23 +00:00
parent b643a436f3
commit a036574a86
1 changed files with 10 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
backends/trtllm/src/backend.rs
View File

@ -20,7 +20,9 @@ use tracing::{instrument, Level, span};
use text_generation_router::{FinishReason, Token}; use text_generation_router::{FinishReason, Token};
use text_generation_router::infer::{Backend, GeneratedText, InferError, InferStreamResponse}; use text_generation_router::infer::{Backend, GeneratedText, InferError, InferStreamResponse};
use text_generation_router::validation::{Chunk, ValidationError, ValidGenerateRequest}; use text_generation_router::validation::{
Chunk, ValidationError, ValidGenerateRequest, ValidParameters,
};
use text_generation_router::validation::ValidationError::UnsupportedModality; use text_generation_router::validation::ValidationError::UnsupportedModality;
use crate::errors::TensorRtLlmBackendError; use crate::errors::TensorRtLlmBackendError;
@ -121,6 +123,11 @@ impl TensorRtLlmBackend {
)); ));
} }
// TODO: Is it really needed? How can it be validated before?
if request.parameters.grammar.is_some() {
return Err(InferError::ValidationError(ValidationError::Grammar));
}
match request.inputs.len() { match request.inputs.len() {
0 => Err(InferError::ValidationError(ValidationError::EmptyInput)), 0 => Err(InferError::ValidationError(ValidationError::EmptyInput)),
2.. => Err(InferError::GenerationError( 2.. => Err(InferError::GenerationError(
@ -308,6 +315,8 @@ impl Backend for TensorRtLlmBackend {
params.top_k, params.top_k,
params.top_p, params.top_p,
params.temperature, params.temperature,
params.repetition_penalty,
params.frequency_penalty,
params.seed, params.seed,
); );