huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-04-27 13:02:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

try-catch to load the cuda extension, quite ugly practice tbh

Browse Source
This commit is contained in:
Felix Marty 2023-07-05 17:53:56 +00:00
parent 620ed7d8aa
commit a6e387404d
1 changed files with 9 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
server/text_generation_server/utils/gptq/quant_linear.py
View File

@ -4,6 +4,15 @@ import torch
import torch.nn as nn import torch.nn as nn
from torch.cuda.amp import custom_bwd, custom_fwd from torch.cuda.amp import custom_bwd, custom_fwd
import torch
from loguru import logger
try:
from custom_kernels.exllama import make_q4, q4_matmul
except Exception as e:
logger.error(f"The CUDA kernels custom_kernels.exllama not installed, got the error: {e}")
try: try:
import triton import triton
import triton.language as tl import triton.language as tl
@ -359,9 +368,6 @@ class QuantLinear(nn.Module):
out = out + self.bias if self.bias is not None else out out = out + self.bias if self.bias is not None else out
return out.reshape(out_shape) return out.reshape(out_shape)
import torch
from custom_kernels.exllama import make_q4, q4_matmul
# Dummy tensor to pass instead of g_idx since there is no way to pass "None" to a C++ extension # Dummy tensor to pass instead of g_idx since there is no way to pass "None" to a C++ extension
none_tensor = torch.empty((1, 1), device = "meta") none_tensor = torch.empty((1, 1), device = "meta")