From bd888f2eb71e01374359681ceba5f29c7b802a34 Mon Sep 17 00:00:00 2001 From: JustSong Date: Wed, 21 Feb 2024 22:19:42 +0800 Subject: [PATCH 01/11] fix: fix prompt token is zero (close #1023) --- relay/controller/text.go | 1 + 1 file changed, 1 insertion(+) diff --git a/relay/controller/text.go b/relay/controller/text.go index 7c49bcce..cc460511 100644 --- a/relay/controller/text.go +++ b/relay/controller/text.go @@ -39,6 +39,7 @@ func RelayTextHelper(c *gin.Context) *model.ErrorWithStatusCode { ratio := modelRatio * groupRatio // pre-consume quota promptTokens := getPromptTokens(textRequest, meta.Mode) + meta.PromptTokens = promptTokens preConsumedQuota, bizErr := preConsumeQuota(ctx, textRequest, promptTokens, ratio, meta) if bizErr != nil { logger.Warnf(ctx, "preConsumeQuota failed: %+v", *bizErr) From 32387d9c208e9cdadac227ce7af5237ee52e0e9a Mon Sep 17 00:00:00 2001 From: JustSong Date: Wed, 21 Feb 2024 22:21:01 +0800 Subject: [PATCH 02/11] fix: fix version is blank --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 94cd8468..ec2f9d43 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,7 +23,7 @@ ADD go.mod go.sum ./ RUN go mod download COPY . . COPY --from=builder /web/build ./web/build -RUN go build -ldflags "-s -w -X 'one-api/common.Version=$(cat VERSION)' -extldflags '-static'" -o one-api +RUN go build -ldflags "-s -w -X 'github.com/songquanpeng/one-api/common.Version=$(cat VERSION)' -extldflags '-static'" -o one-api FROM alpine From 0664bdfda1602aaf36a3baac6435494e6ac9ea86 Mon Sep 17 00:00:00 2001 From: JustSong Date: Sun, 25 Feb 2024 01:53:27 +0800 Subject: [PATCH 03/11] fix: fix build.sh (close #1026) --- .github/workflows/linux-release.yml | 2 +- .github/workflows/macos-release.yml | 2 +- .github/workflows/windows-release.yml | 2 +- web/build.sh | 9 ++++----- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/linux-release.yml b/.github/workflows/linux-release.yml index d93c70ca..98edc471 100644 --- a/.github/workflows/linux-release.yml +++ b/.github/workflows/linux-release.yml @@ -23,7 +23,7 @@ jobs: - uses: actions/setup-node@v3 with: node-version: 16 - - name: Build Frontend (theme default) + - name: Build Frontend env: CI: "" run: | diff --git a/.github/workflows/macos-release.yml b/.github/workflows/macos-release.yml index ce9d1f11..9142609f 100644 --- a/.github/workflows/macos-release.yml +++ b/.github/workflows/macos-release.yml @@ -23,7 +23,7 @@ jobs: - uses: actions/setup-node@v3 with: node-version: 16 - - name: Build Frontend (theme default) + - name: Build Frontend env: CI: "" run: | diff --git a/.github/workflows/windows-release.yml b/.github/workflows/windows-release.yml index 9b1f16ba..c058f41d 100644 --- a/.github/workflows/windows-release.yml +++ b/.github/workflows/windows-release.yml @@ -26,7 +26,7 @@ jobs: - uses: actions/setup-node@v3 with: node-version: 16 - - name: Build Frontend (theme default) + - name: Build Frontend env: CI: "" run: | diff --git a/web/build.sh b/web/build.sh index b3751ff4..252bf943 100644 --- a/web/build.sh +++ b/web/build.sh @@ -1,13 +1,12 @@ #!/bin/sh version=$(cat VERSION) -themes=$(cat THEMES) -IFS=$'\n' +pwd -for theme in $themes; do +while IFS= read -r theme; do echo "Building theme: $theme" - cd $theme + cd "$theme" npm install DISABLE_ESLINT_PLUGIN='true' REACT_APP_VERSION=$version npm run build cd .. -done +done < THEMES From 4c4a873890ef8e91dc89b09465ad0d128a7efba9 Mon Sep 17 00:00:00 2001 From: JustSong Date: Sun, 25 Feb 2024 01:59:40 +0800 Subject: [PATCH 04/11] fix: add an ending line for THEMES --- web/THEMES | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/THEMES b/web/THEMES index b6597eeb..6b0157cb 100644 --- a/web/THEMES +++ b/web/THEMES @@ -1,2 +1,2 @@ default -berry \ No newline at end of file +berry From 87c7c4f0e6f59df96fdf062dd1045aaadc55d205 Mon Sep 17 00:00:00 2001 From: JustSong Date: Sun, 25 Feb 2024 02:07:34 +0800 Subject: [PATCH 05/11] fix: rm history build before building --- web/build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/web/build.sh b/web/build.sh index 252bf943..b59babe4 100644 --- a/web/build.sh +++ b/web/build.sh @@ -5,6 +5,7 @@ pwd while IFS= read -r theme; do echo "Building theme: $theme" + rm -r build/$theme cd "$theme" npm install DISABLE_ESLINT_PLUGIN='true' REACT_APP_VERSION=$version npm run build From 5b78886ad368519616d51662b38522dc2b088590 Mon Sep 17 00:00:00 2001 From: JustSong Date: Sun, 25 Feb 2024 16:53:46 +0800 Subject: [PATCH 06/11] fix: fix i18n --- i18n/en.json | 1 + 1 file changed, 1 insertion(+) diff --git a/i18n/en.json b/i18n/en.json index 774be837..54728e2f 100644 --- a/i18n/en.json +++ b/i18n/en.json @@ -456,6 +456,7 @@ "已绑定的邮箱账户": "Email Account Bound", "用户信息更新成功！": "User information updated successfully!", "模型倍率 %.2f，分组倍率 %.2f": "model rate %.2f, group rate %.2f", + "模型倍率 %.2f，分组倍率 %.2f，补全倍率 %.2f": "model rate %.2f, group rate %.2f, completion rate %.2f", "使用明细（总消耗额度：{renderQuota(stat.quota)}）": "Usage Details (Total Consumption Quota: {renderQuota(stat.quota)})", "用户名称": "User Name", "令牌名称": "Token Name", From f141a37a9e928ff992f57cca02ba6ce7f3059160 Mon Sep 17 00:00:00 2001 From: JustSong Date: Sun, 25 Feb 2024 16:58:14 +0800 Subject: [PATCH 07/11] fix: fix "error update user quota cache: Error 1040: Too many connections" --- model/cache.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model/cache.go b/model/cache.go index 297df153..04a60348 100644 --- a/model/cache.go +++ b/model/cache.go @@ -94,7 +94,7 @@ func CacheUpdateUserQuota(id int) error { if !common.RedisEnabled { return nil } - quota, err := GetUserQuota(id) + quota, err := CacheGetUserQuota(id) if err != nil { return err } From 565ea58e68c10defa6265093149da43b38a3e5fa Mon Sep 17 00:00:00 2001 From: JustSong Date: Sun, 25 Feb 2024 19:01:49 +0800 Subject: [PATCH 08/11] feat: built in retry supported (close #1036, close #770) --- common/helper/helper.go | 10 ++++ controller/relay.go | 99 +++++++++++++++++++++++++++++---------- middleware/auth.go | 2 +- middleware/distributor.go | 55 ++++++++++++---------- middleware/request-id.go | 2 +- 5 files changed, 117 insertions(+), 51 deletions(-) diff --git a/common/helper/helper.go b/common/helper/helper.go index a0d88ec2..babe422b 100644 --- a/common/helper/helper.go +++ b/common/helper/helper.go @@ -137,6 +137,7 @@ func GetUUID() string { } const keyChars = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" +const keyNumbers = "0123456789" func init() { rand.Seed(time.Now().UnixNano()) @@ -168,6 +169,15 @@ func GetRandomString(length int) string { return string(key) } +func GetRandomNumberString(length int) string { + rand.Seed(time.Now().UnixNano()) + key := make([]byte, length) + for i := 0; i < length; i++ { + key[i] = keyNumbers[rand.Intn(len(keyNumbers))] + } + return string(key) +} + func GetTimestamp() int64 { return time.Now().Unix() } diff --git a/controller/relay.go b/controller/relay.go index 6c6d268e..240042b6 100644 --- a/controller/relay.go +++ b/controller/relay.go @@ -1,23 +1,24 @@ package controller import ( + "context" "fmt" "github.com/gin-gonic/gin" "github.com/songquanpeng/one-api/common/config" "github.com/songquanpeng/one-api/common/helper" "github.com/songquanpeng/one-api/common/logger" + "github.com/songquanpeng/one-api/middleware" + dbmodel "github.com/songquanpeng/one-api/model" "github.com/songquanpeng/one-api/relay/constant" "github.com/songquanpeng/one-api/relay/controller" "github.com/songquanpeng/one-api/relay/model" "github.com/songquanpeng/one-api/relay/util" "net/http" - "strconv" ) // https://platform.openai.com/docs/api-reference/chat -func Relay(c *gin.Context) { - relayMode := constant.Path2RelayMode(c.Request.URL.Path) +func relay(c *gin.Context, relayMode int) *model.ErrorWithStatusCode { var err *model.ErrorWithStatusCode switch relayMode { case constant.RelayModeImagesGenerations: @@ -31,32 +32,80 @@ func Relay(c *gin.Context) { default: err = controller.RelayTextHelper(c) } - if err != nil { - requestId := c.GetString(logger.RequestIdKey) - retryTimesStr := c.Query("retry") - retryTimes, _ := strconv.Atoi(retryTimesStr) - if retryTimesStr == "" { - retryTimes = config.RetryTimes + return err +} + +func Relay(c *gin.Context) { + ctx := c.Request.Context() + relayMode := constant.Path2RelayMode(c.Request.URL.Path) + bizErr := relay(c, relayMode) + if bizErr == nil { + return + } + channelId := c.GetInt("channel_id") + lastFailedChannelId := channelId + channelName := c.GetString("channel_name") + group := c.GetString("group") + originalModel := c.GetString("original_model") + go processChannelRelayError(ctx, channelId, channelName, bizErr) + requestId := c.GetString(logger.RequestIdKey) + retryTimes := config.RetryTimes + if !shouldRetry(bizErr.StatusCode) { + logger.Errorf(ctx, "relay error happen, but status code is %d, won't retry in this case", bizErr.StatusCode) + retryTimes = 0 + } + for i := retryTimes; i > 0; i-- { + channel, err := dbmodel.CacheGetRandomSatisfiedChannel(group, originalModel) + if err != nil { + logger.Errorf(ctx, "CacheGetRandomSatisfiedChannel failed: %w", err) + break } - if retryTimes > 0 { - c.Redirect(http.StatusTemporaryRedirect, fmt.Sprintf("%s?retry=%d", c.Request.URL.Path, retryTimes-1)) - } else { - if err.StatusCode == http.StatusTooManyRequests { - err.Error.Message = "当前分组上游负载已饱和，请稍后再试" - } - err.Error.Message = helper.MessageWithRequestId(err.Error.Message, requestId) - c.JSON(err.StatusCode, gin.H{ - "error": err.Error, - }) + logger.Infof(ctx, "using channel #%d to retry (remain times %d)", channel.Id, i) + if channel.Id == lastFailedChannelId { + continue + } + middleware.SetupContextForSelectedChannel(c, channel, originalModel) + bizErr = relay(c, relayMode) + if bizErr == nil { + return } channelId := c.GetInt("channel_id") - logger.Error(c.Request.Context(), fmt.Sprintf("relay error (channel #%d): %s", channelId, err.Message)) - // https://platform.openai.com/docs/guides/error-codes/api-errors - if util.ShouldDisableChannel(&err.Error, err.StatusCode) { - channelId := c.GetInt("channel_id") - channelName := c.GetString("channel_name") - disableChannel(channelId, channelName, err.Message) + lastFailedChannelId = channelId + channelName := c.GetString("channel_name") + go processChannelRelayError(ctx, channelId, channelName, bizErr) + } + if bizErr != nil { + if bizErr.StatusCode == http.StatusTooManyRequests { + bizErr.Error.Message = "当前分组上游负载已饱和，请稍后再试" } + bizErr.Error.Message = helper.MessageWithRequestId(bizErr.Error.Message, requestId) + c.JSON(bizErr.StatusCode, gin.H{ + "error": bizErr.Error, + }) + } +} + +func shouldRetry(statusCode int) bool { + if statusCode == http.StatusTooManyRequests { + return true + } + if statusCode/100 == 5 { + return true + } + if statusCode == http.StatusBadRequest { + return false + } + if statusCode/100 == 2 { + return false + } + return true +} + +func processChannelRelayError(ctx context.Context, channelId int, channelName string, err *model.ErrorWithStatusCode) { + logger.Errorf(ctx, "relay error (channel #%d): %s", channelId, err.Message) + // https://platform.openai.com/docs/guides/error-codes/api-errors + if util.ShouldDisableChannel(&err.Error, err.StatusCode) { + disableChannel(channelId, channelName, err.Message) } } diff --git a/middleware/auth.go b/middleware/auth.go index 42a599d0..9d25f395 100644 --- a/middleware/auth.go +++ b/middleware/auth.go @@ -108,7 +108,7 @@ func TokenAuth() func(c *gin.Context) { c.Set("token_name", token.Name) if len(parts) > 1 { if model.IsAdmin(token.UserId) { - c.Set("channelId", parts[1]) + c.Set("specific_channel_id", parts[1]) } else { abortWithMessage(c, http.StatusForbidden, "普通用户不支持指定渠道") return diff --git a/middleware/distributor.go b/middleware/distributor.go index 704f6236..aeb2796a 100644 --- a/middleware/distributor.go +++ b/middleware/distributor.go @@ -21,8 +21,9 @@ func Distribute() func(c *gin.Context) { userId := c.GetInt("id") userGroup, _ := model.CacheGetUserGroup(userId) c.Set("group", userGroup) + var requestModel string var channel *model.Channel - channelId, ok := c.Get("channelId") + channelId, ok := c.Get("specific_channel_id") if ok { id, err := strconv.Atoi(channelId.(string)) if err != nil { @@ -66,6 +67,7 @@ func Distribute() func(c *gin.Context) { modelRequest.Model = "whisper-1" } } + requestModel = modelRequest.Model channel, err = model.CacheGetRandomSatisfiedChannel(userGroup, modelRequest.Model) if err != nil { message := fmt.Sprintf("当前分组 %s 下对于模型 %s 无可用渠道", userGroup, modelRequest.Model) @@ -77,29 +79,34 @@ func Distribute() func(c *gin.Context) { return } } - c.Set("channel", channel.Type) - c.Set("channel_id", channel.Id) - c.Set("channel_name", channel.Name) - c.Set("model_mapping", channel.GetModelMapping()) - c.Request.Header.Set("Authorization", fmt.Sprintf("Bearer %s", channel.Key)) - c.Set("base_url", channel.GetBaseURL()) - // this is for backward compatibility - switch channel.Type { - case common.ChannelTypeAzure: - c.Set(common.ConfigKeyAPIVersion, channel.Other) - case common.ChannelTypeXunfei: - c.Set(common.ConfigKeyAPIVersion, channel.Other) - case common.ChannelTypeGemini: - c.Set(common.ConfigKeyAPIVersion, channel.Other) - case common.ChannelTypeAIProxyLibrary: - c.Set(common.ConfigKeyLibraryID, channel.Other) - case common.ChannelTypeAli: - c.Set(common.ConfigKeyPlugin, channel.Other) - } - cfg, _ := channel.LoadConfig() - for k, v := range cfg { - c.Set(common.ConfigKeyPrefix+k, v) - } + SetupContextForSelectedChannel(c, channel, requestModel) c.Next() } } + +func SetupContextForSelectedChannel(c *gin.Context, channel *model.Channel, modelName string) { + c.Set("channel", channel.Type) + c.Set("channel_id", channel.Id) + c.Set("channel_name", channel.Name) + c.Set("model_mapping", channel.GetModelMapping()) + c.Set("original_model", modelName) // for retry + c.Request.Header.Set("Authorization", fmt.Sprintf("Bearer %s", channel.Key)) + c.Set("base_url", channel.GetBaseURL()) + // this is for backward compatibility + switch channel.Type { + case common.ChannelTypeAzure: + c.Set(common.ConfigKeyAPIVersion, channel.Other) + case common.ChannelTypeXunfei: + c.Set(common.ConfigKeyAPIVersion, channel.Other) + case common.ChannelTypeGemini: + c.Set(common.ConfigKeyAPIVersion, channel.Other) + case common.ChannelTypeAIProxyLibrary: + c.Set(common.ConfigKeyLibraryID, channel.Other) + case common.ChannelTypeAli: + c.Set(common.ConfigKeyPlugin, channel.Other) + } + cfg, _ := channel.LoadConfig() + for k, v := range cfg { + c.Set(common.ConfigKeyPrefix+k, v) + } +} diff --git a/middleware/request-id.go b/middleware/request-id.go index 7cb66e93..234a93d8 100644 --- a/middleware/request-id.go +++ b/middleware/request-id.go @@ -9,7 +9,7 @@ import ( func RequestId() func(c *gin.Context) { return func(c *gin.Context) { - id := helper.GetTimeString() + helper.GetRandomString(8) + id := helper.GetTimeString() + helper.GetRandomNumberString(8) c.Set(logger.RequestIdKey, id) ctx := context.WithValue(c.Request.Context(), logger.RequestIdKey, id) c.Request = c.Request.WithContext(ctx) From c880b4a9a3db0d156a16e58d34fa5cbc239def10 Mon Sep 17 00:00:00 2001 From: JustSong Date: Sun, 25 Feb 2024 19:17:37 +0800 Subject: [PATCH 09/11] fix: fix missing index in ChatCompletionsStreamResponseChoice (#1037) --- relay/channel/openai/model.go | 1 + 1 file changed, 1 insertion(+) diff --git a/relay/channel/openai/model.go b/relay/channel/openai/model.go index c09f2334..a0aff479 100644 --- a/relay/channel/openai/model.go +++ b/relay/channel/openai/model.go @@ -118,6 +118,7 @@ type ImageResponse struct { } type ChatCompletionsStreamResponseChoice struct { + Index int `json:"index"` Delta struct { Content string `json:"content"` } `json:"delta"` From dc5b781191cfc2b893f581760dda106825e2afdb Mon Sep 17 00:00:00 2001 From: JustSong Date: Sun, 25 Feb 2024 19:47:59 +0800 Subject: [PATCH 10/11] fix: fix stream response id --- relay/channel/aiproxy/main.go | 6 +++--- relay/channel/xunfei/main.go | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/relay/channel/aiproxy/main.go b/relay/channel/aiproxy/main.go index 0d3d0b60..96972407 100644 --- a/relay/channel/aiproxy/main.go +++ b/relay/channel/aiproxy/main.go @@ -53,7 +53,7 @@ func responseAIProxyLibrary2OpenAI(response *LibraryResponse) *openai.TextRespon FinishReason: "stop", } fullTextResponse := openai.TextResponse{ - Id: helper.GetUUID(), + Id: fmt.Sprintf("chatcmpl-%s", helper.GetUUID()), Object: "chat.completion", Created: helper.GetTimestamp(), Choices: []openai.TextResponseChoice{choice}, @@ -66,7 +66,7 @@ func documentsAIProxyLibrary(documents []LibraryDocument) *openai.ChatCompletion choice.Delta.Content = aiProxyDocuments2Markdown(documents) choice.FinishReason = &constant.StopFinishReason return &openai.ChatCompletionsStreamResponse{ - Id: helper.GetUUID(), + Id: fmt.Sprintf("chatcmpl-%s", helper.GetUUID()), Object: "chat.completion.chunk", Created: helper.GetTimestamp(), Model: "", @@ -78,7 +78,7 @@ func streamResponseAIProxyLibrary2OpenAI(response *LibraryStreamResponse) *opena var choice openai.ChatCompletionsStreamResponseChoice choice.Delta.Content = response.Content return &openai.ChatCompletionsStreamResponse{ - Id: helper.GetUUID(), + Id: fmt.Sprintf("chatcmpl-%s", helper.GetUUID()), Object: "chat.completion.chunk", Created: helper.GetTimestamp(), Model: response.Model, diff --git a/relay/channel/xunfei/main.go b/relay/channel/xunfei/main.go index d064b11d..620e808f 100644 --- a/relay/channel/xunfei/main.go +++ b/relay/channel/xunfei/main.go @@ -70,6 +70,7 @@ func responseXunfei2OpenAI(response *ChatResponse) *openai.TextResponse { FinishReason: constant.StopFinishReason, } fullTextResponse := openai.TextResponse{ + Id: fmt.Sprintf("chatcmpl-%s", helper.GetUUID()), Object: "chat.completion", Created: helper.GetTimestamp(), Choices: []openai.TextResponseChoice{choice}, @@ -92,6 +93,7 @@ func streamResponseXunfei2OpenAI(xunfeiResponse *ChatResponse) *openai.ChatCompl choice.FinishReason = &constant.StopFinishReason } response := openai.ChatCompletionsStreamResponse{ + Id: fmt.Sprintf("chatcmpl-%s", helper.GetUUID()), Object: "chat.completion.chunk", Created: helper.GetTimestamp(), Model: "SparkDesk", From 6b27d6659ad801ab48d60e63479bf7c04382d370 Mon Sep 17 00:00:00 2001 From: JustSong Date: Sun, 25 Feb 2024 19:49:22 +0800 Subject: [PATCH 11/11] fix: add role for ChatCompletionsStreamResponseChoice.Delta --- relay/channel/openai/model.go | 1 + 1 file changed, 1 insertion(+) diff --git a/relay/channel/openai/model.go b/relay/channel/openai/model.go index a0aff479..b24485a8 100644 --- a/relay/channel/openai/model.go +++ b/relay/channel/openai/model.go @@ -121,6 +121,7 @@ type ChatCompletionsStreamResponseChoice struct { Index int `json:"index"` Delta struct { Content string `json:"content"` + Role string `json:"role,omitempty"` } `json:"delta"` FinishReason *string `json:"finish_reason,omitempty"` }