test(neuron): avoid using image sha when exporting models

We now manually evaluate the apparent hash of the neuron backend by
combining the hash of the neuron backend directory and Dockerfile.
This new hash is used to identify exported neuron models instead of the
image sha.
This has two benefits:
- it changes less frequently (only hwen the neuron backend changes),
  which means less neuron models being pushed to the hub,
- it can be evaluated locally, meaning that running the tests once
  locally will export the models before the CI uses them.
This commit is contained in:
David Corvoysier 2025-02-21 12:46:57 +00:00
parent 05ca5e4c0f
commit cd477d800c

View File

@ -6,6 +6,7 @@ from tempfile import TemporaryDirectory
import huggingface_hub
import pytest
import docker
import hashlib
import os
import tempfile
@ -50,9 +51,28 @@ MODEL_CONFIGURATIONS = {
}
def get_neuron_backend_hash():
import subprocess
res = subprocess.run(["git", "rev-parse", "--show-toplevel"],
capture_output=True,
text=True)
root_dir = res.stdout.split('\n')[0]
def get_sha(path):
res = subprocess.run(["git", "ls-tree", "HEAD", f"{root_dir}/{path}"],
capture_output=True,
text=True)
# Output of the command is in the form '040000 tree|blob <SHA>\t<path>\n'
sha = res.stdout.split('\t')[0].split(' ')[-1]
return sha.encode()
# We hash both the neuron backends directory and Dockerfile and create a smaller hash out of that
m = hashlib.sha256()
m.update(get_sha('backends/neuron'))
m.update(get_sha('Dockerfile.neuron'))
return m.hexdigest()[:10]
def get_neuron_model_name(config_name: str):
version = get_tgi_docker_image().split(":")[-1]
return f"neuron-tgi-testing-{config_name}-{version}"
return f"neuron-tgi-testing-{config_name}-{get_neuron_backend_hash()}"
def get_tgi_docker_image():