Ecosystem/ai-gateway
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: implement improved headers for anthropic to support 8k outputs (#1654)

Browse Source
This commit is contained in:
Laisky.Cai 2024-07-16 23:48:54 +08:00 committed by GitHub
parent 6209ff9ea9
commit adba54acd3
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 38 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
common/config/config.go
View File

@ -147,7 +147,6 @@ var InitialRootAccessToken = os.Getenv("INITIAL_ROOT_ACCESS_TOKEN")
var GeminiVersion = env.String("GEMINI_VERSION", "v1") var GeminiVersion = env.String("GEMINI_VERSION", "v1")
var OnlyOneLogFile = env.Bool("ONLY_ONE_LOG_FILE", false) var OnlyOneLogFile = env.Bool("ONLY_ONE_LOG_FILE", false)
var RelayProxy = env.String("RELAY_PROXY", "") var RelayProxy = env.String("RELAY_PROXY", "")

13
relay/adaptor/anthropic/adaptor.go
View File

@ -3,12 +3,14 @@ package anthropic
import ( import (
"errors" "errors"
"fmt" "fmt"
"io"
"net/http"
"strings"
"github.com/gin-gonic/gin" "github.com/gin-gonic/gin"
"github.com/songquanpeng/one-api/relay/adaptor" "github.com/songquanpeng/one-api/relay/adaptor"
"github.com/songquanpeng/one-api/relay/meta" "github.com/songquanpeng/one-api/relay/meta"
"github.com/songquanpeng/one-api/relay/model" "github.com/songquanpeng/one-api/relay/model"
"io"
"net/http"
) )
type Adaptor struct { type Adaptor struct {
@ -31,6 +33,13 @@ func (a *Adaptor) SetupRequestHeader(c *gin.Context, req *http.Request, meta *me
} }
req.Header.Set("anthropic-version", anthropicVersion) req.Header.Set("anthropic-version", anthropicVersion)
req.Header.Set("anthropic-beta", "messages-2023-12-15") req.Header.Set("anthropic-beta", "messages-2023-12-15")
// https://x.com/alexalbert__/status/1812921642143900036
// claude-3-5-sonnet can support 8k context
if strings.HasPrefix(meta.ActualModelName, "claude-3-5-sonnet") {
req.Header.Set("anthropic-beta", "max-tokens-3-5-sonnet-2024-07-15")
}
return nil return nil
} }

22
relay/adaptor/vertexai/claude/model.go
View File

@ -4,16 +4,16 @@ import "github.com/songquanpeng/one-api/relay/adaptor/anthropic"
type Request struct { type Request struct {
// AnthropicVersion must be "vertex-2023-10-16" // AnthropicVersion must be "vertex-2023-10-16"
AnthropicVersion string `json:"anthropic_version"` AnthropicVersion string `json:"anthropic_version"`
// Model string `json:"model"` // Model string `json:"model"`
Messages []anthropic.Message `json:"messages"` Messages []anthropic.Message `json:"messages"`
System string `json:"system,omitempty"` System string `json:"system,omitempty"`
MaxTokens int `json:"max_tokens,omitempty"` MaxTokens int `json:"max_tokens,omitempty"`
StopSequences []string `json:"stop_sequences,omitempty"` StopSequences []string `json:"stop_sequences,omitempty"`
Stream bool `json:"stream,omitempty"` Stream bool `json:"stream,omitempty"`
Temperature float64 `json:"temperature,omitempty"` Temperature float64 `json:"temperature,omitempty"`
TopP float64 `json:"top_p,omitempty"` TopP float64 `json:"top_p,omitempty"`
TopK int `json:"top_k,omitempty"` TopK int `json:"top_k,omitempty"`
Tools []anthropic.Tool `json:"tools,omitempty"` Tools []anthropic.Tool `json:"tools,omitempty"`
ToolChoice any `json:"tool_choice,omitempty"` ToolChoice any `json:"tool_choice,omitempty"`
} }

1
relay/adaptor/vertexai/registry.go
View File

@ -32,7 +32,6 @@ func init() {
} }
} }
type innerAIAdapter interface { type innerAIAdapter interface {
ConvertRequest(c *gin.Context, relayMode int, request *model.GeneralOpenAIRequest) (any, error) ConvertRequest(c *gin.Context, relayMode int, request *model.GeneralOpenAIRequest) (any, error)
DoResponse(c *gin.Context, resp *http.Response, meta *meta.Meta) (usage *model.Usage, err *model.ErrorWithStatusCode) DoResponse(c *gin.Context, resp *http.Response, meta *meta.Meta) (usage *model.Usage, err *model.ErrorWithStatusCode)

1
relay/adaptor/vertexai/token.go
View File

@ -26,7 +26,6 @@ type ApplicationDefaultCredentials struct {
UniverseDomain string `json:"universe_domain"` UniverseDomain string `json:"universe_domain"`
} }
var Cache = cache.New(50*time.Minute, 55*time.Minute) var Cache = cache.New(50*time.Minute, 55*time.Minute)
const defaultScope = "https://www.googleapis.com/auth/cloud-platform" const defaultScope = "https://www.googleapis.com/auth/cloud-platform"

2
relay/channeltype/url.go
View File

@ -43,7 +43,7 @@ var ChannelBaseURLs = []string{
"https://api.together.xyz", // 39 "https://api.together.xyz", // 39
"https://ark.cn-beijing.volces.com", // 40 "https://ark.cn-beijing.volces.com", // 40
"https://api.novita.ai/v3/openai", // 41 "https://api.novita.ai/v3/openai", // 41
"", // 42 "", // 42
} }
func init() { func init() {

28
relay/meta/relay_meta.go
View File

@ -10,20 +10,22 @@ import (
) )
type Meta struct { type Meta struct {
Mode int Mode int
ChannelType int ChannelType int
ChannelId int ChannelId int
TokenId int TokenId int
TokenName string TokenName string
UserId int UserId int
Group string Group string
ModelMapping map[string]string ModelMapping map[string]string
BaseURL string BaseURL string
APIKey string APIKey string
APIType int APIType int
Config model.ChannelConfig Config model.ChannelConfig
IsStream bool IsStream bool
// OriginModelName is the model name from the raw user request
OriginModelName string OriginModelName string
// ActualModelName is the model name after mapping
ActualModelName string ActualModelName string
RequestURLPath string RequestURLPath string
PromptTokens int // only for DoResponse PromptTokens int // only for DoResponse