huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-11 20:34:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: use model name as adapter id in chat endpoints

Browse Source
This commit is contained in:
drbh 2024-06-26 23:21:39 +00:00
parent be2d38032a
commit 29a1137409
2 changed files with 6 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
router/src/lib.rs
View File

@ -370,7 +370,7 @@ pub struct CompletionRequest {
/// UNUSED
#[schema(example = "mistralai/Mistral-7B-Instruct-v0.2")]
/// ID of the model to use. See the model endpoint compatibility table for details on which models work with the Chat API.
pub model: String,
pub model: Option<String>,
/// The prompt to generate completions for.
#[schema(example = "What is Deep Learning?")]
@ -706,7 +706,7 @@ impl ChatCompletionChunk {
pub(crate) struct ChatRequest {
#[schema(example = "mistralai/Mistral-7B-Instruct-v0.2")]
/// [UNUSED] ID of the model to use. See the model endpoint compatibility table for details on which models work with the Chat API.
pub model: String,
pub model: Option<String>,
/// A list of messages comprising the conversation so far.
#[schema(example = "[{\"role\": \"user\", \"content\": \"What is Deep Learning?\"}]")]

6
router/src/server.rs
View File

@ -606,6 +606,7 @@ async fn completions(
metrics::increment_counter!("tgi_request_count");
let CompletionRequest {
model,
max_tokens,
seed,
stop,
@ -673,7 +674,7 @@ async fn completions(
seed,
top_n_tokens: None,
grammar: None,
..Default::default()
adapter_id: model.as_ref().filter(|m| *m != "tgi").map(String::from),
},
})
.collect();
@ -1011,6 +1012,7 @@ async fn chat_completions(
let span = tracing::Span::current();
metrics::increment_counter!("tgi_request_count");
let ChatRequest {
model,
logprobs,
max_tokens,
messages,
@ -1116,7 +1118,7 @@ async fn chat_completions(
seed,
top_n_tokens: req.top_logprobs,
grammar,
..Default::default()
adapter_id: model.filter(|m| *m != "tgi").map(String::from),
},
};