From b2a13b92f9c2cf2c8dab5ae7cb11f04cd9474d63 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Wed, 29 Jan 2025 22:24:18 +0100 Subject: [PATCH] Do not fail on missing moe-kernels (Intel-cpu). --- server/text_generation_server/layers/moe/fp8.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/server/text_generation_server/layers/moe/fp8.py b/server/text_generation_server/layers/moe/fp8.py index 7ccddb5b..4d0295f6 100644 --- a/server/text_generation_server/layers/moe/fp8.py +++ b/server/text_generation_server/layers/moe/fp8.py @@ -10,7 +10,11 @@ from text_generation_server.layers.fp8 import ( quant_dtype, normalize_e4m3fn_to_native_float8, ) -from moe_kernels.fused_moe import fused_moe + +try: + from moe_kernels.fused_moe import fused_moe +except Exception: + fused_moe = None class FP8SparseMoELayer(nn.Module):