diff --git a/Cargo.lock b/Cargo.lock index d10efd15..e8a28bf9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2109,6 +2109,7 @@ dependencies = [ "futures", "grpc-metadata", "prost", + "prost-build", "thiserror", "tokio", "tonic", diff --git a/README.md b/README.md index 79651401..938f602a 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,7 @@ to power LLMs api-inference widgets. - Logits warpers (temperature scaling, topk, repetition penalty ...) - Stop sequences - Log probabilities +- Distributed tracing with Open Telemetry ## Officially supported models diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 5d7874ca..3c8d9fcc 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -44,6 +44,8 @@ struct Args { master_port: usize, #[clap(long, env)] json_output: bool, + #[clap(long, env)] + otlp_endpoint: Option, } fn main() -> ExitCode { @@ -62,6 +64,7 @@ fn main() -> ExitCode { master_addr, master_port, json_output, + otlp_endpoint, } = Args::parse(); if json_output { @@ -99,6 +102,7 @@ fn main() -> ExitCode { let status_sender = status_sender.clone(); let shutdown = shutdown.clone(); let shutdown_sender = shutdown_sender.clone(); + let otlp_endpoint = otlp_endpoint.clone(); thread::spawn(move || { shard_manager( model_id, @@ -109,6 +113,7 @@ fn main() -> ExitCode { num_shard, master_addr, master_port, + otlp_endpoint, status_sender, shutdown, shutdown_sender, @@ -174,6 +179,12 @@ fn main() -> ExitCode { argv.push("--json-output".to_string()); } + // OpenTelemetry + if let Some(otlp_endpoint) = otlp_endpoint { + argv.push("--otlp-endpoint".to_string()); + argv.push(otlp_endpoint); + } + let mut webserver = match Popen::create( &argv, PopenConfig { @@ -264,6 +275,7 @@ fn shard_manager( world_size: usize, master_addr: String, master_port: usize, + otlp_endpoint: Option, status_sender: mpsc::Sender, shutdown: Arc>, _shutdown_sender: mpsc::Sender<()>, @@ -286,6 +298,7 @@ fn shard_manager( "--json-output".to_string(), ]; + // Activate tensor parallelism if world_size > 1 { shard_argv.push("--sharded".to_string()); } @@ -294,11 +307,18 @@ fn shard_manager( shard_argv.push("--quantize".to_string()) } + // Model optional revision if let Some(revision) = revision { shard_argv.push("--revision".to_string()); shard_argv.push(revision) } + // OpenTelemetry + if let Some(otlp_endpoint) = otlp_endpoint { + shard_argv.push("--otlp-endpoint".to_string()); + shard_argv.push(otlp_endpoint); + } + let mut env = vec![ ("RANK".into(), rank.to_string().into()), ("WORLD_SIZE".into(), world_size.to_string().into()), diff --git a/router/client/Cargo.toml b/router/client/Cargo.toml index b9a1e52b..7c7ed48e 100644 --- a/router/client/Cargo.toml +++ b/router/client/Cargo.toml @@ -15,4 +15,5 @@ tracing = "^0.1" tracing-error = "^0.2" [build-dependencies] -tonic-build = "^0.8" +tonic-build = "0.8.4" +prost-build = "0.11.6" diff --git a/router/client/build.rs b/router/client/build.rs index a717e1ed..497be545 100644 --- a/router/client/build.rs +++ b/router/client/build.rs @@ -3,12 +3,16 @@ use std::fs; fn main() -> Result<(), Box> { println!("cargo:rerun-if-changed=../../proto/generate.proto"); fs::create_dir("src/pb").unwrap_or(()); + + let mut config = prost_build::Config::new(); + config.protoc_arg("--experimental_allow_proto3_optional"); + tonic_build::configure() .build_client(true) .build_server(false) .out_dir("src/pb") .include_file("mod.rs") - .compile(&["../../proto/generate.proto"], &["../../proto"]) + .compile_with_config(config, &["../../proto/generate.proto"], &["../../proto"]) .unwrap_or_else(|e| panic!("protobuf compilation failed: {e}")); Ok(()) diff --git a/router/client/src/lib.rs b/router/client/src/lib.rs index 2767c605..21fbc1e1 100644 --- a/router/client/src/lib.rs +++ b/router/client/src/lib.rs @@ -17,21 +17,25 @@ use tonic::Status; #[derive(Error, Debug, Clone)] pub enum ClientError { - #[error("Could not connect to Text Generation server: {0:?}")] + #[error("Could not connect to Text Generation server: {0}")] Connection(String), - #[error("Server error: {0:?}")] + #[error("Server error: {0}")] Generation(String), } impl From for ClientError { fn from(err: Status) -> Self { - Self::Generation(err.message().to_string()) + let err = Self::Generation(err.message().to_string()); + tracing::error!("{err}"); + err } } impl From for ClientError { fn from(err: transport::Error) -> Self { - Self::Connection(err.to_string()) + let err = Self::Connection(err.to_string()); + tracing::error!("{err}"); + err } } diff --git a/router/src/infer.rs b/router/src/infer.rs index bb4aa01e..f2716e40 100644 --- a/router/src/infer.rs +++ b/router/src/infer.rs @@ -3,7 +3,6 @@ use crate::validation::{Validation, ValidationError}; use crate::GenerateRequest; use crate::{Entry, Queue, Token}; use nohash_hasher::IntMap; -use opentelemetry::trace::TraceContextExt; use std::future::Future; use std::sync::Arc; use text_generation_client::{ @@ -14,8 +13,7 @@ use tokio::sync::{mpsc, Notify, Semaphore, TryAcquireError}; use tokio::time::Instant; use tokio_stream::wrappers::UnboundedReceiverStream; use tokio_stream::StreamExt; -use tracing::{info_span, instrument, Instrument}; -use tracing_opentelemetry::OpenTelemetrySpanExt; +use tracing::{info_span, instrument, Instrument, Span}; /// Inference struct #[derive(Clone)] @@ -78,7 +76,14 @@ impl Infer { ) -> Result>, InferError> { // Limit concurrent requests by acquiring a permit from the semaphore // This permit will live as long as Entry - let permit = self.clone().limit_concurrent_requests.try_acquire_owned()?; + let permit = self + .clone() + .limit_concurrent_requests + .try_acquire_owned() + .map_err(|err| { + tracing::error!("{err}"); + err + })?; // Validate request let valid_request = self.validation.validate(request).await?; @@ -90,9 +95,9 @@ impl Infer { self.queue.append(Entry { request: valid_request, response_tx, - parent_span: info_span!("entry"), + span: Span::current(), batch_span: None, - time: Instant::now(), + queue_time: Instant::now(), batch_time: None, _permit: permit, }); @@ -166,7 +171,9 @@ impl Infer { start, }) } else { - Err(InferError::IncompleteGeneration) + let err = InferError::IncompleteGeneration; + tracing::error!("{err}"); + Err(err) } } } @@ -235,13 +242,13 @@ async fn batching_task( } } } - let next_batch_span = info_span!("batch"); + // Create span for this batch to add context to inference calls + let next_batch_span = info_span!(parent: None, "batch"); entries.iter_mut().for_each(|(_, entry)| { - // Create a new span for this entry/batch tuple - let entry_batch_span = info_span!(parent: &entry.parent_span, "infer"); - // Add link to span - entry_batch_span - .add_link(next_batch_span.context().span().span_context().clone()); + // Create a new span to link the batch back to this entry + let entry_batch_span = info_span!(parent: &entry.span, "infer"); + // Add relationship + entry_batch_span.follows_from(&next_batch_span); // Update entry entry.batch_span = Some(entry_batch_span); }); @@ -268,7 +275,7 @@ async fn wrap_future( } // If we have an error, we discard the whole batch Err(err) => { - send_error(err, entries); + send_errors(err, entries); None } } @@ -276,12 +283,17 @@ async fn wrap_future( /// Send errors to Infer for all `entries` #[instrument] -fn send_error(error: ClientError, entries: &mut IntMap) { +fn send_errors(error: ClientError, entries: &mut IntMap) { entries.drain().for_each(|(_, entry)| { + // Create and enter a span to link this function back to the entry + let _send_error_span = info_span!(parent: entry.batch_span.as_ref().expect("batch_span is None. This is a bug."), "send_error").entered(); + let err = InferError::GenerationError(error.to_string()); + tracing::error!("{err}"); + // unwrap_or is valid here as we don't care if the receiver is gone. entry .response_tx - .send(Err(InferError::GenerationError(error.to_string()))) + .send(Err(err)) .unwrap_or(()); }); } @@ -296,6 +308,7 @@ fn send_generations(generations: Vec, entries: &mut IntMap, entries: &mut IntMap>, - /// Request Span - pub parent_span: Span, - /// Batch Span + /// Span that will live as long as entry + pub span: Span, + /// Span for every inference batch + /// This span will only live as long as one prefill/decode pub batch_span: Option, - /// Instant when this entry was created - pub time: Instant, + /// Instant when this entry was queued + pub queue_time: Instant, /// Instant when this entry was added to a batch pub batch_time: Option, /// Permit @@ -49,6 +48,7 @@ impl Queue { } /// Append an entry to the queue + #[instrument(skip(self))] pub(crate) fn append(&self, entry: Entry) { // Send append command to the background task managing the state // Unwrap is safe here @@ -58,6 +58,7 @@ impl Queue { } // Get the next batch + #[instrument(skip(self))] pub(crate) async fn next_batch( &self, min_size: Option, @@ -142,7 +143,10 @@ impl State { } } - let next_batch_span = info_span!("batch"); + // Create span for this batch to add context to inference calls + let next_batch_span = info_span!(parent: None, "batch"); + next_batch_span.follows_from(&Span::current()); + let next_batch_size = min(self.entries.len(), max_size); let mut batch_requests = Vec::with_capacity(next_batch_size); @@ -153,10 +157,10 @@ impl State { self.entries .drain(..next_batch_size) .for_each(|(id, mut entry)| { - // Create a new span for this entry/batch tuple - let entry_batch_span = info_span!(parent: &entry.parent_span, "infer"); - // Add link to span - entry_batch_span.add_link(next_batch_span.context().span().span_context().clone()); + // Create a new span to link the batch back to this entry + let entry_batch_span = info_span!(parent: &entry.span, "infer"); + // Add relationship + entry_batch_span.follows_from(&next_batch_span); // Update entry entry.batch_span = Some(entry_batch_span); @@ -229,9 +233,9 @@ mod tests { }, }, response_tx, - parent_span: info_span!("entry"), + span: info_span!("entry"), batch_span: None, - time: Instant::now(), + queue_time: Instant::now(), batch_time: None, _permit: permit, } diff --git a/router/src/server.rs b/router/src/server.rs index bcfddd9c..09be5f22 100644 --- a/router/src/server.rs +++ b/router/src/server.rs @@ -88,10 +88,7 @@ async fn generate( // Inference let details = req.0.parameters.details; - let response = infer.generate(req.0).await.map_err(|err| { - tracing::error!("{}", err.to_string()); - err - })?; + let response = infer.generate(req.0).await?; // Token details let details = match details { @@ -265,19 +262,17 @@ async fn generate_stream( } } } - // Trace and yield error + // yield error Err(err) => { error = true; - tracing::error!("{}", err.to_string()); yield Ok(Event::from(err)) } } } }, - // Trace and yield error + // yield error Err(err) => { error = true; - tracing::error!("{}", err.to_string()); yield Ok(Event::from(err)) } } @@ -285,7 +280,7 @@ async fn generate_stream( // Skip if we already sent an error if !end_reached && !error { let err = InferError::IncompleteGeneration; - tracing::error!("{}", err.to_string()); + tracing::error!("{err}"); yield Ok(Event::from(err)) } }; diff --git a/router/src/validation.rs b/router/src/validation.rs index 42eb9b12..b3e05fc0 100644 --- a/router/src/validation.rs +++ b/router/src/validation.rs @@ -105,7 +105,12 @@ fn validation_worker( while let Some((request, response_tx, parent_span)) = receiver.blocking_recv() { parent_span.in_scope(|| { response_tx - .send(validate(request, &tokenizer, max_input_length, &mut rng)) + .send( + validate(request, &tokenizer, max_input_length, &mut rng).map_err(|err| { + tracing::error!("{err}"); + err + }), + ) .unwrap_or(()) }) } diff --git a/server/Makefile b/server/Makefile index 6961178b..74ce5144 100644 --- a/server/Makefile +++ b/server/Makefile @@ -1,6 +1,6 @@ gen-server: # Compile protos - pip install grpcio-tools==1.49.1 --no-cache-dir + pip install grpcio-tools==1.51.1 --no-cache-dir mkdir text_generation/pb || true python -m grpc_tools.protoc -I../proto --python_out=text_generation/pb --grpc_python_out=text_generation/pb ../proto/generate.proto find text_generation/pb/ -type f -name "*.py" -print0 -exec sed -i -e 's/^\(import.*pb2\)/from . \1/g' {} \; diff --git a/server/poetry.lock b/server/poetry.lock index 39a67168..0e0655cb 100644 --- a/server/poetry.lock +++ b/server/poetry.lock @@ -225,7 +225,7 @@ dev = ["Sphinx (>=4.1.1)", "black (>=19.10b0)", "colorama (>=0.3.4)", "docutils [[package]] name = "numpy" -version = "1.24.1" +version = "1.24.2" description = "Fundamental package for array computing in Python" category = "main" optional = false @@ -511,7 +511,7 @@ torch = ["torch"] [[package]] name = "setuptools" -version = "67.0.0" +version = "67.2.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" category = "main" optional = false @@ -622,7 +622,7 @@ bnb = ["bitsandbytes"] [metadata] lock-version = "1.1" python-versions = "^3.9" -content-hash = "1a1fe3fd152c3b1a48908f4e3302b92278a3180d6fe5f39c27dc2b7b34d30dbf" +content-hash = "f3cab6881b52045770a90ec9be7415a0ee499d9e980892d544f68073700cf321" [metadata.files] accelerate = [ @@ -874,34 +874,34 @@ loguru = [ {file = "loguru-0.6.0.tar.gz", hash = "sha256:066bd06758d0a513e9836fd9c6b5a75bfb3fd36841f4b996bc60b547a309d41c"}, ] numpy = [ - {file = "numpy-1.24.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:179a7ef0889ab769cc03573b6217f54c8bd8e16cef80aad369e1e8185f994cd7"}, - {file = "numpy-1.24.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b09804ff570b907da323b3d762e74432fb07955701b17b08ff1b5ebaa8cfe6a9"}, - {file = "numpy-1.24.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f1b739841821968798947d3afcefd386fa56da0caf97722a5de53e07c4ccedc7"}, - {file = "numpy-1.24.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e3463e6ac25313462e04aea3fb8a0a30fb906d5d300f58b3bc2c23da6a15398"}, - {file = "numpy-1.24.1-cp310-cp310-win32.whl", hash = "sha256:b31da69ed0c18be8b77bfce48d234e55d040793cebb25398e2a7d84199fbc7e2"}, - {file = "numpy-1.24.1-cp310-cp310-win_amd64.whl", hash = "sha256:b07b40f5fb4fa034120a5796288f24c1fe0e0580bbfff99897ba6267af42def2"}, - {file = "numpy-1.24.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7094891dcf79ccc6bc2a1f30428fa5edb1e6fb955411ffff3401fb4ea93780a8"}, - {file = "numpy-1.24.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:28e418681372520c992805bb723e29d69d6b7aa411065f48216d8329d02ba032"}, - {file = "numpy-1.24.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e274f0f6c7efd0d577744f52032fdd24344f11c5ae668fe8d01aac0422611df1"}, - {file = "numpy-1.24.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0044f7d944ee882400890f9ae955220d29b33d809a038923d88e4e01d652acd9"}, - {file = "numpy-1.24.1-cp311-cp311-win32.whl", hash = "sha256:442feb5e5bada8408e8fcd43f3360b78683ff12a4444670a7d9e9824c1817d36"}, - {file = "numpy-1.24.1-cp311-cp311-win_amd64.whl", hash = "sha256:de92efa737875329b052982e37bd4371d52cabf469f83e7b8be9bb7752d67e51"}, - {file = "numpy-1.24.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b162ac10ca38850510caf8ea33f89edcb7b0bb0dfa5592d59909419986b72407"}, - {file = "numpy-1.24.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:26089487086f2648944f17adaa1a97ca6aee57f513ba5f1c0b7ebdabbe2b9954"}, - {file = "numpy-1.24.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:caf65a396c0d1f9809596be2e444e3bd4190d86d5c1ce21f5fc4be60a3bc5b36"}, - {file = "numpy-1.24.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b0677a52f5d896e84414761531947c7a330d1adc07c3a4372262f25d84af7bf7"}, - {file = "numpy-1.24.1-cp38-cp38-win32.whl", hash = "sha256:dae46bed2cb79a58d6496ff6d8da1e3b95ba09afeca2e277628171ca99b99db1"}, - {file = "numpy-1.24.1-cp38-cp38-win_amd64.whl", hash = "sha256:6ec0c021cd9fe732e5bab6401adea5a409214ca5592cd92a114f7067febcba0c"}, - {file = "numpy-1.24.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:28bc9750ae1f75264ee0f10561709b1462d450a4808cd97c013046073ae64ab6"}, - {file = "numpy-1.24.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:84e789a085aabef2f36c0515f45e459f02f570c4b4c4c108ac1179c34d475ed7"}, - {file = "numpy-1.24.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e669fbdcdd1e945691079c2cae335f3e3a56554e06bbd45d7609a6cf568c700"}, - {file = "numpy-1.24.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ef85cf1f693c88c1fd229ccd1055570cb41cdf4875873b7728b6301f12cd05bf"}, - {file = "numpy-1.24.1-cp39-cp39-win32.whl", hash = "sha256:87a118968fba001b248aac90e502c0b13606721b1343cdaddbc6e552e8dfb56f"}, - {file = "numpy-1.24.1-cp39-cp39-win_amd64.whl", hash = "sha256:ddc7ab52b322eb1e40521eb422c4e0a20716c271a306860979d450decbb51b8e"}, - {file = "numpy-1.24.1-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:ed5fb71d79e771ec930566fae9c02626b939e37271ec285e9efaf1b5d4370e7d"}, - {file = "numpy-1.24.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ad2925567f43643f51255220424c23d204024ed428afc5aad0f86f3ffc080086"}, - {file = "numpy-1.24.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:cfa1161c6ac8f92dea03d625c2d0c05e084668f4a06568b77a25a89111621566"}, - {file = "numpy-1.24.1.tar.gz", hash = "sha256:2386da9a471cc00a1f47845e27d916d5ec5346ae9696e01a8a34760858fe9dd2"}, + {file = "numpy-1.24.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:eef70b4fc1e872ebddc38cddacc87c19a3709c0e3e5d20bf3954c147b1dd941d"}, + {file = "numpy-1.24.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e8d2859428712785e8a8b7d2b3ef0a1d1565892367b32f915c4a4df44d0e64f5"}, + {file = "numpy-1.24.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6524630f71631be2dabe0c541e7675db82651eb998496bbe16bc4f77f0772253"}, + {file = "numpy-1.24.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a51725a815a6188c662fb66fb32077709a9ca38053f0274640293a14fdd22978"}, + {file = "numpy-1.24.2-cp310-cp310-win32.whl", hash = "sha256:2620e8592136e073bd12ee4536149380695fbe9ebeae845b81237f986479ffc9"}, + {file = "numpy-1.24.2-cp310-cp310-win_amd64.whl", hash = "sha256:97cf27e51fa078078c649a51d7ade3c92d9e709ba2bfb97493007103c741f1d0"}, + {file = "numpy-1.24.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7de8fdde0003f4294655aa5d5f0a89c26b9f22c0a58790c38fae1ed392d44a5a"}, + {file = "numpy-1.24.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4173bde9fa2a005c2c6e2ea8ac1618e2ed2c1c6ec8a7657237854d42094123a0"}, + {file = "numpy-1.24.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4cecaed30dc14123020f77b03601559fff3e6cd0c048f8b5289f4eeabb0eb281"}, + {file = "numpy-1.24.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9a23f8440561a633204a67fb44617ce2a299beecf3295f0d13c495518908e910"}, + {file = "numpy-1.24.2-cp311-cp311-win32.whl", hash = "sha256:e428c4fbfa085f947b536706a2fc349245d7baa8334f0c5723c56a10595f9b95"}, + {file = "numpy-1.24.2-cp311-cp311-win_amd64.whl", hash = "sha256:557d42778a6869c2162deb40ad82612645e21d79e11c1dc62c6e82a2220ffb04"}, + {file = "numpy-1.24.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d0a2db9d20117bf523dde15858398e7c0858aadca7c0f088ac0d6edd360e9ad2"}, + {file = "numpy-1.24.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c72a6b2f4af1adfe193f7beb91ddf708ff867a3f977ef2ec53c0ffb8283ab9f5"}, + {file = "numpy-1.24.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c29e6bd0ec49a44d7690ecb623a8eac5ab8a923bce0bea6293953992edf3a76a"}, + {file = "numpy-1.24.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2eabd64ddb96a1239791da78fa5f4e1693ae2dadc82a76bc76a14cbb2b966e96"}, + {file = "numpy-1.24.2-cp38-cp38-win32.whl", hash = "sha256:e3ab5d32784e843fc0dd3ab6dcafc67ef806e6b6828dc6af2f689be0eb4d781d"}, + {file = "numpy-1.24.2-cp38-cp38-win_amd64.whl", hash = "sha256:76807b4063f0002c8532cfeac47a3068a69561e9c8715efdad3c642eb27c0756"}, + {file = "numpy-1.24.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4199e7cfc307a778f72d293372736223e39ec9ac096ff0a2e64853b866a8e18a"}, + {file = "numpy-1.24.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:adbdce121896fd3a17a77ab0b0b5eedf05a9834a18699db6829a64e1dfccca7f"}, + {file = "numpy-1.24.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:889b2cc88b837d86eda1b17008ebeb679d82875022200c6e8e4ce6cf549b7acb"}, + {file = "numpy-1.24.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f64bb98ac59b3ea3bf74b02f13836eb2e24e48e0ab0145bbda646295769bd780"}, + {file = "numpy-1.24.2-cp39-cp39-win32.whl", hash = "sha256:63e45511ee4d9d976637d11e6c9864eae50e12dc9598f531c035265991910468"}, + {file = "numpy-1.24.2-cp39-cp39-win_amd64.whl", hash = "sha256:a77d3e1163a7770164404607b7ba3967fb49b24782a6ef85d9b5f54126cc39e5"}, + {file = "numpy-1.24.2-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:92011118955724465fb6853def593cf397b4a1367495e0b59a7e69d40c4eb71d"}, + {file = "numpy-1.24.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9006288bcf4895917d02583cf3411f98631275bc67cce355a7f39f8c14338fa"}, + {file = "numpy-1.24.2-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:150947adbdfeceec4e5926d956a06865c1c690f2fd902efede4ca6fe2e657c3f"}, + {file = "numpy-1.24.2.tar.gz", hash = "sha256:003a9f530e880cb2cd177cba1af7220b9aa42def9c4afc2a2fc3ee6be7eb2b22"}, ] nvidia-cublas-cu11 = [ {file = "nvidia_cublas_cu11-11.10.3.66-py3-none-manylinux1_x86_64.whl", hash = "sha256:d32e4d75f94ddfb93ea0a5dda08389bcc65d8916a25cb9f37ac89edaeed3bded"}, @@ -1089,8 +1089,8 @@ safetensors = [ {file = "safetensors-0.2.8.tar.gz", hash = "sha256:2720b20a6a38c799dca79bd76caeeac2f7df585a9d4f7d59fa7e28eff9ccb27f"}, ] setuptools = [ - {file = "setuptools-67.0.0-py3-none-any.whl", hash = "sha256:9d790961ba6219e9ff7d9557622d2fe136816a264dd01d5997cfc057d804853d"}, - {file = "setuptools-67.0.0.tar.gz", hash = "sha256:883131c5b6efa70b9101c7ef30b2b7b780a4283d5fc1616383cdf22c83cbefe6"}, + {file = "setuptools-67.2.0-py3-none-any.whl", hash = "sha256:16ccf598aab3b506593c17378473978908a2734d7336755a8769b480906bec1c"}, + {file = "setuptools-67.2.0.tar.gz", hash = "sha256:b440ee5f7e607bb8c9de15259dba2583dd41a38879a7abc1d43a71c59524da48"}, ] tomli = [ {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, diff --git a/server/pyproject.toml b/server/pyproject.toml index 5c8c20ac..d3a8c112 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -27,7 +27,7 @@ opentelemetry-instrumentation-grpc = "^0.36b0" bnb = ["bitsandbytes"] [tool.poetry.group.dev.dependencies] -grpcio-tools = "^1.49.1" +grpcio-tools = "^1.51.1" pytest = "^7.2.0" [build-system] diff --git a/server/text_generation/tracing.py b/server/text_generation/tracing.py index fc90a8ae..bf03c379 100644 --- a/server/text_generation/tracing.py +++ b/server/text_generation/tracing.py @@ -10,10 +10,7 @@ from opentelemetry.sdk.resources import Resource from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import ( BatchSpanProcessor, - ConsoleSpanExporter, - SimpleSpanProcessor, ) -from typing import Optional class UDSOpenTelemetryAioServerInterceptor(OpenTelemetryAioServerInterceptor):