huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-10 20:04:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add ignore_eos_token

Browse Source
This commit is contained in:
Chirag Jain 2023-08-21 14:06:17 +05:30
parent 9f18f4c006
commit ecbc0a0f4a
No known key found for this signature in database
GPG Key ID: D6A7538AE0A6D9AF
3 changed files with 42 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
Dockerfile-compile-router Normal file
View File

@ -0,0 +1,36 @@
# Rust builder
FROM --platform=linux/amd64 lukemathwalker/cargo-chef:latest-rust-1.69 AS chef
WORKDIR /usr/src
ARG CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse
FROM chef as planner
COPY Cargo.toml Cargo.toml
COPY rust-toolchain.toml rust-toolchain.toml
COPY proto proto
COPY benchmark benchmark
COPY router router
COPY launcher launcher
RUN cargo chef prepare --recipe-path recipe.json
FROM --platform=linux/amd64 chef AS builder
ARG GIT_SHA
ARG DOCKER_LABEL
RUN PROTOC_ZIP=protoc-21.12-linux-x86_64.zip && \
curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v21.12/$PROTOC_ZIP && \
unzip -o $PROTOC_ZIP -d /usr/local bin/protoc && \
unzip -o $PROTOC_ZIP -d /usr/local 'include/*' && \
rm -f $PROTOC_ZIP
COPY --from=planner /usr/src/recipe.json recipe.json
RUN cargo chef cook --release --recipe-path recipe.json
COPY Cargo.toml Cargo.toml
COPY rust-toolchain.toml rust-toolchain.toml
COPY proto proto
COPY benchmark benchmark
COPY router router
COPY launcher launcher
RUN cargo build --release

4
router/src/lib.rs
View File

@ -135,6 +135,9 @@ pub(crate) struct GenerateParameters {
example = "null"
)]
pub seed: Option<u64>,
#[serde(default)]
#[schema(default = "false")]
pub ignore_eos_token: bool
}
fn default_max_new_tokens() -> u32 {
@ -158,6 +161,7 @@ fn default_parameters() -> GenerateParameters {
details: false,
decoder_input_details: false,
seed: None,
ignore_eos_token: false,
}
}

3
router/src/validation.rs
View File

@ -142,6 +142,7 @@ impl Validation {
seed,
watermark,
decoder_input_details,
ignore_eos_token,
..
} = request.parameters;
@ -251,7 +252,7 @@ impl Validation {
let stopping_parameters = StoppingCriteriaParameters {
max_new_tokens,
stop_sequences,
ignore_eos_token: false,
ignore_eos_token: ignore_eos_token,
};
metrics::histogram!("tgi_request_max_new_tokens", max_new_tokens as f64);