mirror of
https://github.com/huggingface/text-generation-inference.git
synced 2025-04-20 06:12:07 +00:00
f9e561eced
* style * update torch * ix issues * fix clone * revert mkl * added custom PA * style * fix style * style * hide env vart * fix mixtral model * add skinny kernel and merge fixes * fixed style * fix issue for sliding window models * addressed review comments * fix import * improved error messag * updated default value * remove import * fix imports after rebase * float16 dep * improve dockerfile * cleaned dockerfile
33 lines
904 B
Python
33 lines
904 B
Python
from setuptools import setup
|
|
from torch.utils.cpp_extension import BuildExtension, CUDAExtension
|
|
import torch
|
|
|
|
extra_cuda_cflags = []
|
|
extra_cflags = []
|
|
if torch.version.hip:
|
|
extra_cflags = ["-DLEGACY_HIPBLAS_DIRECT=ON"]
|
|
extra_cuda_cflags = ["-DLEGACY_HIPBLAS_DIRECT=ON"]
|
|
|
|
extra_compile_args = {
|
|
"cxx": extra_cflags,
|
|
"nvcc": extra_cuda_cflags,
|
|
}
|
|
|
|
setup(
|
|
name="exllama_kernels",
|
|
ext_modules=[
|
|
CUDAExtension(
|
|
name="exllama_kernels",
|
|
sources=[
|
|
"exllama_kernels/exllama_ext.cpp",
|
|
"exllama_kernels/cuda_buffers.cu",
|
|
"exllama_kernels/cuda_func/column_remap.cu",
|
|
"exllama_kernels/cuda_func/q4_matmul.cu",
|
|
"exllama_kernels/cuda_func/q4_matrix.cu",
|
|
],
|
|
extra_compile_args=extra_compile_args,
|
|
)
|
|
],
|
|
cmdclass={"build_ext": BuildExtension},
|
|
)
|