diff --git a/controller/relay.go b/controller/relay.go index ced2a901..692373c4 100644 --- a/controller/relay.go +++ b/controller/relay.go @@ -6,6 +6,7 @@ import ( "encoding/json" "fmt" "github.com/gin-gonic/gin" + "github.com/pkoukk/tiktoken-go" "io" "net/http" "one-api/common" @@ -44,6 +45,13 @@ type StreamResponse struct { } `json:"choices"` } +var tokenEncoder, _ = tiktoken.GetEncoding("cl100k_base") + +func countToken(text string) int { + token := tokenEncoder.Encode(text, nil, nil) + return len(token) +} + func Relay(c *gin.Context) { err := relayHelper(c) if err != nil { @@ -64,6 +72,7 @@ func relayHelper(c *gin.Context) error { if channelType == common.ChannelTypeCustom { baseURL = c.GetString("base_url") } + var textRequest TextRequest if consumeQuota { requestBody, err := io.ReadAll(c.Request.Body) if err != nil { @@ -73,7 +82,6 @@ func relayHelper(c *gin.Context) error { if err != nil { return err } - var textRequest TextRequest err = json.Unmarshal(requestBody, &textRequest) if err != nil { return err @@ -112,7 +120,12 @@ func relayHelper(c *gin.Context) error { if consumeQuota { quota := 0 if isStream { - quota = int(float64(len(streamResponseText)) * common.BytesNumber2Quota) + var text string + for _, message := range textRequest.Messages { + text += fmt.Sprintf("%s: %s\n", message.Role, message.Content) + } + text += fmt.Sprintf("%s: %s\n", "assistant", streamResponseText) + quota = countToken(text) + 3 } else { quota = textResponse.Usage.TotalTokens } diff --git a/go.mod b/go.mod index a84c243f..85ff2f4a 100644 --- a/go.mod +++ b/go.mod @@ -25,6 +25,7 @@ require ( github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/dlclark/regexp2 v1.8.1 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect @@ -44,6 +45,7 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml/v2 v2.0.7 // indirect + github.com/pkoukk/tiktoken-go v0.1.1 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.11 // indirect golang.org/x/arch v0.3.0 // indirect diff --git a/go.sum b/go.sum index 4bd964b1..f11b9780 100644 --- a/go.sum +++ b/go.sum @@ -14,6 +14,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/dlclark/regexp2 v1.8.1 h1:6Lcdwya6GjPUNsBct8Lg/yRPwMhABj269AAzdGSiR+0= +github.com/dlclark/regexp2 v1.8.1/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/gin-contrib/cors v1.4.0 h1:oJ6gwtUl3lqV0WEIwM/LxPF1QZ5qe2lGWdY2+bz7y0g= github.com/gin-contrib/cors v1.4.0/go.mod h1:bs9pNM0x/UsmHPBWT2xZz9ROh8xYjYkiURUfmBoMlcs= @@ -119,6 +121,8 @@ github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZO github.com/pelletier/go-toml/v2 v2.0.7 h1:muncTPStnKRos5dpVKULv2FVd4bMOhNePj9CjgDb8Us= github.com/pelletier/go-toml/v2 v2.0.7/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pkoukk/tiktoken-go v0.1.1 h1:jtkYlIECjyM9OW1w4rjPmTohK4arORP9V25y6TM6nXo= +github.com/pkoukk/tiktoken-go v0.1.1/go.mod h1:boMWvk9pQCOTx11pgu0DrIdrAKgQzzJKUP6vLXaz7Rw= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=