huggingface/text-generation-inference
5
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2025-04-22 15:32:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix test_pass_through_tokenizer (#117)

Browse Source 
Co-authored-by: Sylwester Fraczek <sfraczek@habana.ai>
This commit is contained in:
Sylwester Fraczek 2024-04-09 09:30:47 +02:00 committed by GitHub
parent d957e32601
commit 757c12dbac
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 11 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
server/tests/utils/test_tokens.py
View File

@ -10,6 +10,16 @@ from text_generation_server.utils.tokens import (
)
from transformers import AutoTokenizer
import pytest
@pytest.fixture
def skip_tokenizer_env_var():
import os
os.environ["SKIP_TOKENIZER_IN_TGI"] = "true"
yield
del os.environ['SKIP_TOKENIZER_IN_TGI']
def test_stop_sequence_criteria():
criteria = StopSequenceCriteria("/test;")
@ -71,8 +81,7 @@ def test_batch_top_tokens():
assert topn_tok_logprobs[4] == [-1, -2, -3, -3, -4]
def test_pass_through_tokenizer():
def test_pass_through_tokenizer(skip_tokenizer_env_var):
tokenizer = AutoTokenizer.from_pretrained(
'meta-llama/Llama-2-7b-chat-hf',
revision=None,