text-generation-inference/server/Makefile-marlin

marlin_commit := 2f6d7c10e124b3c5fa29ff8d77d568bd7af3274c

build-marlin:
	if [ ! -d 'marlin' ]; then \
		pip install -U ninja packaging --no-cache-dir && \
		git clone https://github.com/IST-DASLab/marlin.git marlin; \
	fi
	cd marlin  && git fetch && git checkout $(marlin_commit) && python setup.py build

install-marlin: build-marlin
	cd marlin  && git fetch && git checkout $(marlin_commit) && pip install -e .
Add support for Marlin-quantized models This change adds support for Marlin-quantized models. Marlin is an FP16xINT4 matmul kernel, which provides good speedups decoding batches of 16-32 tokens. It supports quantized models with symmetric quantization, groupsize -1 or 128, and 4-bit. Tested with: - Llama 2 - Llama 3 - Phi 3 2024-06-05 08:14:40 +00:00			`marlin_commit := 2f6d7c10e124b3c5fa29ff8d77d568bd7af3274c`

marlin: improve build 2024-06-06 11:25:56 +00:00			`build-marlin:`
			`if [ ! -d 'marlin' ]; then \`
			`pip install -U ninja packaging --no-cache-dir && \`
			`git clone https://github.com/IST-DASLab/marlin.git marlin; \`
			`fi`
			`cd marlin && git fetch && git checkout $(marlin_commit) && python setup.py build`
Add support for Marlin-quantized models This change adds support for Marlin-quantized models. Marlin is an FP16xINT4 matmul kernel, which provides good speedups decoding batches of 16-32 tokens. It supports quantized models with symmetric quantization, groupsize -1 or 128, and 4-bit. Tested with: - Llama 2 - Llama 3 - Phi 3 2024-06-05 08:14:40 +00:00
			`install-marlin: build-marlin`
marlin: improve build 2024-06-06 11:25:56 +00:00			`cd marlin && git fetch && git checkout $(marlin_commit) && pip install -e .`