From 6170b91d1c6705f8937ca05b580cb77139a2e8ec Mon Sep 17 00:00:00 2001 From: NongMO <31816449+nongmo677@users.noreply.github.com> Date: Sat, 27 Apr 2024 15:47:27 +0800 Subject: [PATCH] feat: support for the ollama vision model (#1376) * feat: support for the ollama vision model `llava` model, pass test * Update main.go format code * chore: remove useless log --------- Co-authored-by: nongqiqin Co-authored-by: JustSong --- relay/adaptor/ollama/main.go | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/relay/adaptor/ollama/main.go b/relay/adaptor/ollama/main.go index b08eb0ca..c5fe08e6 100644 --- a/relay/adaptor/ollama/main.go +++ b/relay/adaptor/ollama/main.go @@ -13,6 +13,7 @@ import ( "github.com/gin-gonic/gin" "github.com/songquanpeng/one-api/common" + "github.com/songquanpeng/one-api/common/image" "github.com/songquanpeng/one-api/common/logger" "github.com/songquanpeng/one-api/relay/adaptor/openai" "github.com/songquanpeng/one-api/relay/constant" @@ -32,9 +33,22 @@ func ConvertRequest(request model.GeneralOpenAIRequest) *ChatRequest { Stream: request.Stream, } for _, message := range request.Messages { + openaiContent := message.ParseContent() + var imageUrls []string + var contentText string + for _, part := range openaiContent { + switch part.Type { + case model.ContentTypeText: + contentText = part.Text + case model.ContentTypeImageURL: + _, data, _ := image.GetImageFromUrl(part.ImageURL.Url) + imageUrls = append(imageUrls, data) + } + } ollamaRequest.Messages = append(ollamaRequest.Messages, Message{ Role: message.Role, - Content: message.StringContent(), + Content: contentText, + Images: imageUrls, }) } return &ollamaRequest