huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-09-11 20:34:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Small fixes

Browse Source
This commit is contained in:
Daniël de Kok 2024-11-18 14:49:59 +00:00
parent 3eb6c1ccf8
commit 05f98efc9d
3 changed files with 6 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
integration-tests/models/test_compressed_tensors_w8a8_int.py
View File

@ -37,7 +37,9 @@ async def test_compressed_tensors_w8a8_int(
assert response == response_snapshot
@pytest.mark.release
@pytest.mark.asyncio
@pytest.mark.private
async def test_compressed_tensors_w8a8_int_all_params(
compressed_tensors_w8a8_int, response_snapshot
):

2
integration-tests/models/test_compressed_tensors_w8a8_int_dynamic_weight.py
View File

@ -39,7 +39,9 @@ async def test_compressed_tensors_w8a8_int_dynamic_weight(
assert response == response_snapshot
@pytest.mark.release
@pytest.mark.asyncio
@pytest.mark.private
async def test_compressed_tensors_w8a8_int_dynamic_weight_all_params(
compressed_tensors_w8a8_int_dynamic_weight, response_snapshot
):

4
server/text_generation_server/layers/compressed_tensors/w8a8_int.py
View File

@ -51,10 +51,10 @@ class W8A8IntLoader(WeightsLoader):
def scale_to_str(scale):
return "static" if scale else "dynamic"
def symmetric_to_string(symmetric):
def symmetric_to_str(symmetric):
return "symmetric" if symmetric else "asymmetric"
return f"{self.__class__.__name__} (w8a8 int, input: dynamic/{symmetric_to_string(self.input_symmetric)}, weight: {scale_to_str(self.load_weight_scale)}/symmetric))"
return f"{self.__class__.__name__} (w8a8 int, input: dynamic/{symmetric_to_str(self.input_symmetric)}, weight: {scale_to_str(self.load_weight_scale)}/symmetric))"
def get_weights(self, weights: "Weights", prefix: str):
w = weights.get_tensor(f"{prefix}.weight", to_dtype=False)