huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-12 12:54:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix to not ignore HUGGINGFACE_HUB_CACHE in cache

Browse Source
This commit is contained in:
ErikKaumk 2024-07-12 14:35:51 +02:00
parent 8511669cb2
commit 57c313df9c
1 changed files with 11 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
router/src/main.rs
View File

@ -210,7 +210,11 @@ async fn main() -> Result<(), RouterError> {
} }
let api = if use_api { let api = if use_api {
if std::env::var("HF_HUB_OFFLINE") == Ok("1".to_string()) { if std::env::var("HF_HUB_OFFLINE") == Ok("1".to_string()) {
let cache = Cache::default(); let cache = std::env::var("HUGGINGFACE_HUB_CACHE")
.map_err(|_| ())
.map(|cache_dir| Cache::new(cache_dir.into()))
.unwrap_or_else(|_| Cache::default());
tracing::warn!("Offline mode active using cache defaults"); tracing::warn!("Offline mode active using cache defaults");
Type::Cache(cache) Type::Cache(cache)
} else { } else {
@ -291,6 +295,10 @@ async fn main() -> Result<(), RouterError> {
) )
} }
}; };
println!("tokenizer_filename: {:?}", tokenizer_filename);
let config: Option<Config> = config_filename.and_then(|filename| { let config: Option<Config> = config_filename.and_then(|filename| {
std::fs::read_to_string(filename) std::fs::read_to_string(filename)
.ok() .ok()
@ -348,6 +356,8 @@ async fn main() -> Result<(), RouterError> {
tracing::warn!("Rust input length validation and truncation is disabled"); tracing::warn!("Rust input length validation and truncation is disabled");
} }
println!("Using config {config:?}");
// if pipeline-tag == text-generation we default to return_full_text = true // if pipeline-tag == text-generation we default to return_full_text = true
let compat_return_full_text = match &model_info.pipeline_tag { let compat_return_full_text = match &model_info.pipeline_tag {
None => { None => {