backend(vllm): initial commit

Cargo.lock

@@ -1,6 +1,6 @@
 # This file is automatically @generated by Cargo.
 # It is not intended for manual editing.
-version = 3
+version = 4
 
 [[package]]
 name = "addr2line"
@@ -4443,6 +4443,10 @@ dependencies = [
  "tracing",
 ]
 
+[[package]]
+name = "text-generation-backends-vllm"
+version = "3.0.2-dev0"
+
 [[package]]
 name = "text-generation-benchmark"
 version = "3.0.2-dev0"

Cargo.toml

@@ -5,15 +5,18 @@ members = [
     "backends/v3",
     "backends/grpc-metadata",
     "backends/trtllm",
+    "backends/vllm",
     "launcher",
-    "router"
+    "router",
 ]
+
 default-members = [
     "benchmark",
     "backends/v2",
     "backends/v3",
     "backends/grpc-metadata",
     # "backends/trtllm",
+    # "backends/vllm",
     "launcher",
     "router"
 ]
@@ -33,7 +36,7 @@ metrics = { version = "0.23.0" }
 metrics-exporter-prometheus = { version = "0.15.1", features = [] }
 minijinja = { version = "2.2.0", features = ["json"] }
 minijinja-contrib = { version = "2.0.2", features = ["pycompat"] }
-pyo3 = { version = "0.22.2", features = ["auto-initialize"] }
+pyo3 = { version = "0.23", features = ["auto-initialize"] }
 
 [profile.release]
 incremental = true

backends/vllm/Cargo.toml

@@ -0,0 +1,10 @@
+[package]
+name = "text-generation-backends-vllm"
+version.workspace = true
+edition.workspace = true
+authors.workspace = true
+homepage.workspace = true
+
+[dependencies]
+pyo3 = "0.23"
+pyo3-asyncio = "0.20"

backends/vllm/src/main.rs

@@ -0,0 +1,6 @@
+use pyo3::prelude::*;
+
+#[pyo3_asyncio::tokio::main(flavor = "multi_thread")]
+async fn main() {
+    println!("Hello, world!");
+}