From 2c90b0575b8d00a5b3a8799740e31a1fcb682228 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Thu, 30 Jan 2025 16:28:55 +0100 Subject: [PATCH] Scoring func `softmax` is the only one that works. --- server/text_generation_server/layers/moe/gptq_marlin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/text_generation_server/layers/moe/gptq_marlin.py b/server/text_generation_server/layers/moe/gptq_marlin.py index d909f397..014a90dc 100644 --- a/server/text_generation_server/layers/moe/gptq_marlin.py +++ b/server/text_generation_server/layers/moe/gptq_marlin.py @@ -72,7 +72,7 @@ class GPTQMarlinSparseMoELayer(nn.Module): scoring_func: Optional[str] = None, e_score_correction_bias: Optional[float] = None, ): - assert scoring_func is None, "scoring func is not handled" + assert scoring_func == "softmax", f"scoring func {scoring_func} is not handled" assert e_score_correction_bias is None, "scoring correction bias is not handled" super().__init__()