diff --git a/README.md b/README.md index f835156c..a3258c32 100644 --- a/README.md +++ b/README.md @@ -87,15 +87,15 @@ Maximum sequence length is controlled by two arguments: - `--max-total-tokens` is the maximum possible total length of the sequence (input and output). Default value is `4096`. Maximum batch size is controlled by two arguments: -- For prefill operation, please set `--max-prefill-total-tokens` as `bs * max-input-tokens`, where `bs` is your expected maximum prefill batch size. +- For prefill operation, please set `--max-batch-prefill-tokens` as `bs * max-input-tokens`, where `bs` is your expected maximum prefill batch size. - For decode operation, please set `--max-batch-total-tokens` as `bs * max-total-tokens`, where `bs` is your expected maximum decode batch size. - Please note that batch size will be always padded to the nearest multiplication of `BATCH_BUCKET_SIZE` and `PREFILL_BATCH_BUCKET_SIZE`. -To ensure greatest performance results, at the begginging of each server run, warmup is performed. It's designed to cover major recompilations while using HPU Graphs. It creates queries with all possible input shapes, based on provided parameters (described in this section) and runs basic TGI operations on them (prefill, decode, concatenate). +To ensure greatest performance results, at the beginning of each server run, warmup is performed. It's designed to cover major recompilations while using HPU Graphs. It creates queries with all possible input shapes, based on provided parameters (described in this section) and runs basic TGI operations on them (prefill, decode, concatenate). Except those already mentioned, there are other parameters that need to be properly adjusted to improve performance or memory usage: -- `PAD_SEQUENCE_TO_MULTIPLE_OF` determines sizes of input legnth buckets. Since warmup creates several graphs for each bucket, it's important to adjust that value proportionally to input sequence length. Otherwise, some out of memory issues can be observed. +- `PAD_SEQUENCE_TO_MULTIPLE_OF` determines sizes of input length buckets. Since warmup creates several graphs for each bucket, it's important to adjust that value proportionally to input sequence length. Otherwise, some out of memory issues can be observed. - `ENABLE_HPU_GRAPH` enables HPU graphs usage, which is crucial for performance results. Recommended value to keep is `true` . For more information and documentation about Text Generation Inference, checkout [the README](https://github.com/huggingface/text-generation-inference#text-generation-inference) of the original repo. @@ -118,7 +118,7 @@ Additional hints to quantize model for TGI when using `run_lm_eval.py`: ## Currently supported configurations Not all features of TGI are currently supported as this is still a work in progress. -Currently supported and validated configurations (other configurations are not guaranted to work or ensure reasonable performance): +Currently supported and validated configurations (other configurations are not guaranteed to work or ensure reasonable performance): ### LLama 7b BF16 on 1 Gaudi2 card diff --git a/server/text_generation_server/server.py b/server/text_generation_server/server.py index f52d801c..b8905d71 100644 --- a/server/text_generation_server/server.py +++ b/server/text_generation_server/server.py @@ -19,6 +19,7 @@ from text_generation_server.interceptor import ExceptionInterceptor from text_generation_server.models import Model, get_model from text_generation_server.pb import generate_pb2_grpc, generate_pb2 from text_generation_server.tracing import UDSOpenTelemetryAioServerInterceptor +from text_generation_server.utils.version import is_driver_compatible, MIN_TGI_GAUDI_SYNAPSE_VERSION class SignalHandler: @@ -53,6 +54,7 @@ class TextGenerationService(generate_pb2_grpc.TextGenerationServiceServicer): # Force inference mode for the lifetime of TextGenerationService # self._inference_mode_raii_guard = torch._C._InferenceMode(True) + async def Info(self, request, context): return self.model.info @@ -164,6 +166,9 @@ def serve( dtype: Optional[str] = None, trust_remote_code: bool = False, ): + if not is_driver_compatible(): + logger.warning(f"Current Synapse version is lower than the minimum version supported: {MIN_TGI_GAUDI_SYNAPSE_VERSION}, this could result in failures") + unix_socket_template = "unix://{}-{}" logger.info("Server:server_inner: sharded ={}".format(sharded)) diff --git a/server/text_generation_server/utils/version.py b/server/text_generation_server/utils/version.py new file mode 100644 index 00000000..a72a9ea7 --- /dev/null +++ b/server/text_generation_server/utils/version.py @@ -0,0 +1,12 @@ +from optimum.habana.utils import get_driver_version +from packaging.version import Version + +MIN_TGI_GAUDI_SYNAPSE_VERSION=Version("1.16.0") + + +def is_driver_compatible(): + driver_version = get_driver_version() + if driver_version is not None: + if driver_version < MIN_TGI_GAUDI_SYNAPSE_VERSION: + return False + return True \ No newline at end of file