From 3c02262f29fba3bf1096f22017f70d59ff4daa4d Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Thu, 23 Nov 2023 15:42:48 +0000 Subject: [PATCH] Reduce race condition on file system for test --- router/src/lib.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/router/src/lib.rs b/router/src/lib.rs index 560b8f74..d6df2f56 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -300,15 +300,22 @@ mod tests { use tokenizers::Tokenizer; pub(crate) async fn get_tokenizer() -> Tokenizer { - if !std::path::Path::new("tokenizer.json").exists() { + let filename = std::path::Path::new("tokenizer.json"); + if !filename.exists() { let content = reqwest::get("https://huggingface.co/gpt2/raw/main/tokenizer.json") .await .unwrap() .bytes() .await .unwrap(); - let mut file = std::fs::File::create("tokenizer.json").unwrap(); + let tmp_filename = "tokenizer.json.temp"; + let mut file = std::fs::File::create(tmp_filename).unwrap(); file.write_all(&content).unwrap(); + // Re-check if another process has written this file maybe. + if !filename.exists() { + std::fs::rename(tmp_filename, filename).unwrap() + } + } Tokenizer::from_file("tokenizer.json").unwrap() }