From c03a7b3560ebdf1aff90c94e1c2ffe613af6c3d6 Mon Sep 17 00:00:00 2001 From: "Wang, Yi A" Date: Wed, 14 Aug 2024 17:49:05 -0700 Subject: [PATCH] update doc with intel cpu part Signed-off-by: Wang, Yi A --- docs/source/installation_intel.md | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/docs/source/installation_intel.md b/docs/source/installation_intel.md index b3843490..16919142 100644 --- a/docs/source/installation_intel.md +++ b/docs/source/installation_intel.md @@ -12,7 +12,24 @@ volume=$PWD/data # share a volume with the Docker container to avoid downloading docker run --rm --privileged --cap-add=sys_nice \ --device=/dev/dri \ --ipc=host --shm-size 1g --net host -v $volume:/data \ - ghcr.io/huggingface/text-generation-inference:2.2.0-intel \ + ghcr.io/huggingface/text-generation-inference:latest-intel-xpu \ + --model-id $model --cuda-graphs 0 +``` + +# Using TGI with Intel CPUs + +Intel® Extension for PyTorch (IPEX) also provides further optimizations for Intel CPUs. The IPEX provides optimization operations such as flash attention, page attention, Add + LayerNorm, ROPE and more. + +On a server powered by Intel CPU, TGI can be launched with the following command: + +```bash +model=teknium/OpenHermes-2.5-Mistral-7B +volume=$PWD/data # share a volume with the Docker container to avoid downloading weights every run + +docker run --rm --privileged --cap-add=sys_nice \ + --device=/dev/dri \ + --ipc=host --shm-size 1g --net host -v $volume:/data \ + ghcr.io/huggingface/text-generation-inference:latest-intel-cpu \ --model-id $model --cuda-graphs 0 ```