Ecosystem/ai-gateway
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Support Ali TTS

Browse Source
This commit is contained in:
mo 2023-12-19 22:47:28 +08:00
parent 366b82128f
commit a6e14cadf3
4 changed files with 580 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
common/model-ratio.go
View File

@ -90,6 +90,49 @@ var ModelRatio = map[string]float64{
"qwen-turbo": 0.8572, // ￥0.012 / 1k tokens
"qwen-plus": 10, // ￥0.14 / 1k tokens
"text-embedding-v1": 0.05, // ￥0.0007 / 1k tokens
"sambert-zhichu-v1": 0.777, // todo
"sambert-zhiwei-v1": 0.777, // todo
"sambert-zhixiang-v1": 0.777, // todo
"sambert-zhide-v1": 0.777, // todo
"sambert-zhijia-v1": 0.777, // todo
"sambert-zhinan-v1": 0.777, // todo
"sambert-zhiqi-v1": 0.777, // todo
"sambert-zhiqian-v1": 0.777, // todo
"sambert-zhiru-v1": 0.777, // todo
"sambert-zhimiao-emo-v1": 0.777, // todo
"sambert-zhida-v1": 0.777, // todo
"sambert-zhifei-v1": 0.777, // todo
"sambert-zhigui-v1": 0.777, // todo
"sambert-zhihao-v1": 0.777, // todo
"sambert-zhijing-v1": 0.777, // todo
"sambert-zhilun-v1": 0.777, // todo
"sambert-zhimao-v1": 0.777, // todo
"sambert-zhiming-v1": 0.777, // todo
"sambert-zhimo-v1": 0.777, // todo
"sambert-zhina-v1": 0.777, // todo
"sambert-zhishu-v1": 0.777, // todo
"sambert-zhishuo-v1": 0.777, // todo
"sambert-zhistella-v1": 0.777, // todo
"sambert-zhiting-v1": 0.777, // todo
"sambert-zhixiao-v1": 0.777, // todo
"sambert-zhiya-v1": 0.777, // todo
"sambert-zhiye-v1": 0.777, // todo
"sambert-zhiying-v1": 0.777, // todo
"sambert-zhiyuan-v1": 0.777, // todo
"sambert-zhiyue-v1": 0.777, // todo
"sambert-camila-v1": 0.777, // todo
"sambert-perla-v1": 0.777, // todo
"sambert-indah-v1": 0.777, // todo
"sambert-clara-v1": 0.777, // todo
"sambert-hanna-v1": 0.777, // todo
"sambert-beth-v1": 0.777, // todo
"sambert-betty-v1": 0.777, // todo
"sambert-cally-v1": 0.777, // todo
"sambert-cindy-v1": 0.777, // todo
"sambert-eva-v1": 0.777, // todo
"sambert-donna-v1": 0.777, // todo
"sambert-brian-v1": 0.777, // todo
"sambert-waan-v1": 0.777, // todo
"SparkDesk": 1.2858, // ￥0.018 / 1k tokens
"360GPT_S2_V9": 0.8572, // ¥0.012 / 1k tokens
"embedding-bert-512-v1": 0.0715, // ¥0.001 / 1k tokens

379
controller/model.go
View File

@ -486,6 +486,385 @@ func init() {
Root: "text-embedding-v1",
Parent: nil,
},
{
Id: "sambert-zhichu-v1",
Object: "model",
Created: 1702621721,
OwnedBy: "ali",
Permission: permission,
Root: "sambert-zhichu-v1",
Parent: nil,
},
{
Id: "sambert-zhiwei-v1",
Object: "model",
Created: 1702621721,
OwnedBy: "ali",
Permission: permission,
Root: "sambert-zhiwei-v1",
Parent: nil,
},
{
Id: "sambert-zhixiang-v1",
Object: "model",
Created: 1702621721,
OwnedBy: "ali",
Permission: permission,
Root: "sambert-zhixiang-v1",
Parent: nil,
},
{
Id: "sambert-zhide-v1",
Object: "model",
Created: 1702621721,
OwnedBy: "ali",
Permission: permission,
Root: "sambert-zhide-v1",
Parent: nil,
},
{
Id: "sambert-zhijia-v1",
Object: "model",
Created: 1702621721,
OwnedBy: "ali",
Permission: permission,
Root: "sambert-zhijia-v1",
Parent: nil,
},
{
Id: "sambert-zhinan-v1",
Object: "model",
Created: 1702621721,
OwnedBy: "ali",
Permission: permission,
Root: "sambert-zhinan-v1",
Parent: nil,
},
{
Id: "sambert-zhiqi-v1",
Object: "model",
Created: 1702621721,
OwnedBy: "ali",
Permission: permission,
Root: "sambert-zhiqi-v1",
Parent: nil,
},
{
Id: "sambert-zhiqian-v1",
Object: "model",
Created: 1702621721,
OwnedBy: "ali",
Permission: permission,
Root: "sambert-zhiqian-v1",
Parent: nil,
},
{
Id: "sambert-zhiru-v1",
Object: "model",
Created: 1702621721,
OwnedBy: "ali",
Permission: permission,
Root: "sambert-zhiru-v1",
Parent: nil,
},
{
Id: "sambert-zhimiao-emo-v1",
Object: "model",
Created: 1702621721,
OwnedBy: "ali",
Permission: permission,
Root: "sambert-zhimiao-emo-v1",
Parent: nil,
},
{
Id: "sambert-zhida-v1",
Object: "model",
Created: 1702621721,
OwnedBy: "ali",
Permission: permission,
Root: "sambert-zhida-v1",
Parent: nil,
},
{
Id: "sambert-zhifei-v1",
Object: "model",
Created: 1702621721,
OwnedBy: "ali",
Permission: permission,
Root: "sambert-zhifei-v1",
Parent: nil,
},
{
Id: "sambert-zhigui-v1",
Object: "model",
Created: 1702621721,
OwnedBy: "ali",
Permission: permission,
Root: "sambert-zhigui-v1",
Parent: nil,
},
{
Id: "sambert-zhihao-v1",
Object: "model",
Created: 1702621721,
OwnedBy: "ali",
Permission: permission,
Root: "sambert-zhihao-v1",
Parent: nil,
},
{
Id: "sambert-zhijing-v1",
Object: "model",
Created: 1702621721,
OwnedBy: "ali",
Permission: permission,
Root: "sambert-zhijing-v1",
Parent: nil,
},
{
Id: "sambert-zhilun-v1",
Object: "model",
Created: 1702621721,
OwnedBy: "ali",
Permission: permission,
Root: "sambert-zhilun-v1",
Parent: nil,
},
{
Id: "sambert-zhimao-v1",
Object: "model",
Created: 1702621721,
OwnedBy: "ali",
Permission: permission,
Root: "sambert-zhimao-v1",
Parent: nil,
},
{
Id: "sambert-zhiming-v1",
Object: "model",
Created: 1702621721,
OwnedBy: "ali",
Permission: permission,
Root: "sambert-zhiming-v1",
Parent: nil,
},
{
Id: "sambert-zhimo-v1",
Object: "model",
Created: 1702621721,
OwnedBy: "ali",
Permission: permission,
Root: "sambert-zhimo-v1",
Parent: nil,
},
{
Id: "sambert-zhina-v1",
Object: "model",
Created: 1702621721,
OwnedBy: "ali",
Permission: permission,
Root: "sambert-zhina-v1",
Parent: nil,
},
{
Id: "sambert-zhishu-v1",
Object: "model",
Created: 1702621721,
OwnedBy: "ali",
Permission: permission,
Root: "sambert-zhishu-v1",
Parent: nil,
},
{
Id: "sambert-zhishuo-v1",
Object: "model",
Created: 1702621721,
OwnedBy: "ali",
Permission: permission,
Root: "sambert-zhishuo-v1",
Parent: nil,
},
{
Id: "sambert-zhistella-v1",
Object: "model",
Created: 1702621721,
OwnedBy: "ali",
Permission: permission,
Root: "sambert-zhistella-v1",
Parent: nil,
},
{
Id: "sambert-zhiting-v1",
Object: "model",
Created: 1702621721,
OwnedBy: "ali",
Permission: permission,
Root: "sambert-zhiting-v1",
Parent: nil,
},
{
Id: "sambert-zhixiao-v1",
Object: "model",
Created: 1702621721,
OwnedBy: "ali",
Permission: permission,
Root: "sambert-zhixiao-v1",
Parent: nil,
},
{
Id: "sambert-zhiya-v1",
Object: "model",
Created: 1702621721,
OwnedBy: "ali",
Permission: permission,
Root: "sambert-zhiya-v1",
Parent: nil,
},
{
Id: "sambert-zhiye-v1",
Object: "model",
Created: 1702621721,
OwnedBy: "ali",
Permission: permission,
Root: "sambert-zhiye-v1",
Parent: nil,
},
{
Id: "sambert-zhiying-v1",
Object: "model",
Created: 1702621721,
OwnedBy: "ali",
Permission: permission,
Root: "sambert-zhiying-v1",
Parent: nil,
},
{
Id: "sambert-zhiyuan-v1",
Object: "model",
Created: 1702621721,
OwnedBy: "ali",
Permission: permission,
Root: "sambert-zhiyuan-v1",
Parent: nil,
},
{
Id: "sambert-zhiyue-v1",
Object: "model",
Created: 1702621721,
OwnedBy: "ali",
Permission: permission,
Root: "sambert-zhiyue-v1",
Parent: nil,
},
{
Id: "sambert-camila-v1",
Object: "model",
Created: 1702621721,
OwnedBy: "ali",
Permission: permission,
Root: "sambert-camila-v1",
Parent: nil,
},
{
Id: "sambert-perla-v1",
Object: "model",
Created: 1702621721,
OwnedBy: "ali",
Permission: permission,
Root: "sambert-perla-v1",
Parent: nil,
},
{
Id: "sambert-indah-v1",
Object: "model",
Created: 1702621721,
OwnedBy: "ali",
Permission: permission,
Root: "sambert-indah-v1",
Parent: nil,
},
{
Id: "sambert-clara-v1",
Object: "model",
Created: 1702621721,
OwnedBy: "ali",
Permission: permission,
Root: "sambert-clara-v1",
Parent: nil,
},
{
Id: "sambert-hanna-v1",
Object: "model",
Created: 1702621721,
OwnedBy: "ali",
Permission: permission,
Root: "sambert-hanna-v1",
Parent: nil,
},
{
Id: "sambert-beth-v1",
Object: "model",
Created: 1702621721,
OwnedBy: "ali",
Permission: permission,
Root: "sambert-beth-v1",
Parent: nil,
},
{
Id: "sambert-cally-v1",
Object: "model",
Created: 1702621721,
OwnedBy: "ali",
Permission: permission,
Root: "sambert-cally-v1",
Parent: nil,
},
{
Id: "sambert-cindy-v1",
Object: "model",
Created: 1702621721,
OwnedBy: "ali",
Permission: permission,
Root: "sambert-cindy-v1",
Parent: nil,
},
{
Id: "sambert-eva-v1",
Object: "model",
Created: 1702621721,
OwnedBy: "ali",
Permission: permission,
Root: "sambert-eva-v1",
Parent: nil,
},
{
Id: "sambert-donna-v1",
Object: "model",
Created: 1702621721,
OwnedBy: "ali",
Permission: permission,
Root: "sambert-donna-v1",
Parent: nil,
},
{
Id: "sambert-brian-v1",
Object: "model",
Created: 1702621721,
OwnedBy: "ali",
Permission: permission,
Root: "sambert-brian-v1",
Parent: nil,
},
{
Id: "sambert-waan-v1",
Object: "model",
Created: 1702621721,
OwnedBy: "ali",
Permission: permission,
Root: "sambert-waan-v1",
Parent: nil,
},
{
Id: "SparkDesk",
Object: "model",

128
controller/relay-ali.go
View File

@ -4,6 +4,8 @@ import (
"bufio"
"encoding/json"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
"github.com/gorilla/websocket"
"io"
"net/http"
"one-api/common"
@ -35,6 +37,61 @@ type AliChatRequest struct {
Parameters AliParameters `json:"parameters,omitempty"`
}
type AliTaskResponse struct {
StatusCode int `json:"status_code,omitempty"`
RequestId string `json:"request_id,omitempty"`
Code string `json:"code,omitempty"`
Message string `json:"message,omitempty"`
Output struct {
TaskId string `json:"task_id,omitempty"`
TaskStatus string `json:"task_status,omitempty"`
Code string `json:"code,omitempty"`
Message string `json:"message,omitempty"`
Results []struct {
B64Image string `json:"b64_image,omitempty"`
Url string `json:"url,omitempty"`
Code string `json:"code,omitempty"`
Message string `json:"message,omitempty"`
} `json:"results,omitempty"`
TaskMetrics struct {
Total int `json:"TOTAL,omitempty"`
Succeeded int `json:"SUCCEEDED,omitempty"`
Failed int `json:"FAILED,omitempty"`
} `json:"task_metrics,omitempty"`
} `json:"output,omitempty"`
Usage Usage `json:"usage"`
}
type AliHeader struct {
Action string `json:"action,omitempty"`
Streaming string `json:"streaming,omitempty"`
TaskID string `json:"task_id,omitempty"`
Event string `json:"event,omitempty"`
}
type AliPayload struct {
Model string `json:"model,omitempty"`
Task string `json:"task,omitempty"`
TaskGroup string `json:"task_group,omitempty"`
Function string `json:"function,omitempty"`
Parameters struct {
SampleRate int `json:"sample_rate,omitempty"`
Rate float64 `json:"rate,omitempty"`
Format string `json:"format,omitempty"`
} `json:"parameters,omitempty"`
Input struct {
Text string `json:"text,omitempty"`
} `json:"input,omitempty"`
Usage struct {
Characters int `json:"characters,omitempty"`
} `json:"usage,omitempty"`
}
type AliWSSMessage struct {
Header AliHeader `json:"header,omitempty"`
Payload AliPayload `json:"payload,omitempty"`
}
type AliEmbeddingRequest struct {
Model string `json:"model"`
Input struct {
@ -119,6 +176,23 @@ func requestOpenAI2Ali(request GeneralOpenAIRequest) *AliChatRequest {
}
}
func requestOpenAI2AliTTS(request TextToSpeechRequest) *AliWSSMessage {
var ttsRequest AliWSSMessage
ttsRequest.Header.Action = "run-task"
ttsRequest.Header.Streaming = "out"
ttsRequest.Header.TaskID = uuid.New().String()
ttsRequest.Payload.Function = "SpeechSynthesizer"
ttsRequest.Payload.Input.Text = request.Input
ttsRequest.Payload.Model = request.Model
ttsRequest.Payload.Parameters.Format = request.ResponseFormat
//ttsRequest.Payload.Parameters.SampleRate = 48000
ttsRequest.Payload.Parameters.Rate = request.Speed
ttsRequest.Payload.Task = "tts"
ttsRequest.Payload.TaskGroup = "audio"
return &ttsRequest
}
func embeddingRequestOpenAI2Ali(request GeneralOpenAIRequest) *AliEmbeddingRequest {
return &AliEmbeddingRequest{
Model: "text-embedding-v1",
@ -327,3 +401,57 @@ func aliHandler(c *gin.Context, resp *http.Response) (*OpenAIErrorWithStatusCode
_, err = c.Writer.Write(jsonResponse)
return nil, &fullTextResponse.Usage
}
func aliTTSHandler(c *gin.Context, req TextToSpeechRequest) (*OpenAIErrorWithStatusCode, *Usage) {
Authorization := c.Request.Header.Get("Authorization")
baseURL := "wss://dashscope.aliyuncs.com/api-ws/v1/inference"
var usage Usage
conn, _, err := websocket.DefaultDialer.Dial(baseURL, http.Header{"Authorization": {Authorization}})
if err != nil {
return errorWrapper(err, "wss_conn_failed", http.StatusInternalServerError), nil
}
defer conn.Close()
message := requestOpenAI2AliTTS(req)
if err := conn.WriteJSON(message); err != nil {
return errorWrapper(err, "wss_write_msg_failed", http.StatusInternalServerError), nil
}
const chunkSize = 1024
for {
messageType, audioData, err := conn.ReadMessage()
if err != nil {
if err == io.EOF {
break
}
return errorWrapper(err, "wss_read_msg_failed", http.StatusInternalServerError), nil
}
var msg AliWSSMessage
switch messageType {
case websocket.TextMessage:
err = json.Unmarshal(audioData, &msg)
if msg.Header.Event == "task-finished" {
usage.TotalTokens = msg.Payload.Usage.Characters
return nil, &usage
}
case websocket.BinaryMessage:
for i := 0; i < len(audioData); i += chunkSize {
end := i + chunkSize
if end > len(audioData) {
end = len(audioData)
}
chunk := audioData[i:end]
_, writeErr := c.Writer.Write(chunk)
if writeErr != nil {
return errorWrapper(writeErr, "write_audio_failed", http.StatusInternalServerError), nil
}
}
}
}
return nil, &usage
}

34
controller/relay-audio.go
View File

@ -90,6 +90,12 @@ func relayAudioHelper(c *gin.Context, relayMode int) *OpenAIErrorWithStatusCode
}
}
apiType := APITypeOpenAI
switch channelType {
case common.ChannelTypeAli:
apiType = APITypeAli
}
baseURL := common.ChannelBaseURLs[channelType]
requestURL := c.Request.URL.String()
if c.GetString("base_url") != "" {
@ -103,11 +109,33 @@ func relayAudioHelper(c *gin.Context, relayMode int) *OpenAIErrorWithStatusCode
fullRequestURL = fmt.Sprintf("%s/openai/deployments/%s/audio/transcriptions?api-version=%s", baseURL, audioModel, apiVersion)
}
quotaDelta := 0
defer func(ctx context.Context) {
go postConsumeQuota(ctx, tokenId, quotaDelta, quota, userId, channelId, modelRatio, groupRatio, audioModel, tokenName)
}(c.Request.Context())
requestBody := &bytes.Buffer{}
_, err = io.Copy(requestBody, c.Request.Body)
if err != nil {
return errorWrapper(err, "new_request_body_failed", http.StatusInternalServerError)
}
switch apiType {
case APITypeAli:
if relayMode == RelayModeAudioSpeech {
err, usage := aliTTSHandler(c, ttsRequest)
if err != nil {
return err
}
if usage != nil {
quota = usage.TotalTokens
}
return nil
}
}
c.Request.Body = io.NopCloser(bytes.NewBuffer(requestBody.Bytes()))
responseFormat := c.DefaultPostForm("response_format", "json")
@ -195,10 +223,8 @@ func relayAudioHelper(c *gin.Context, relayMode int) *OpenAIErrorWithStatusCode
}
return relayErrorHandler(resp)
}
quotaDelta := quota - preConsumedQuota
defer func(ctx context.Context) {
go postConsumeQuota(ctx, tokenId, quotaDelta, quota, userId, channelId, modelRatio, groupRatio, audioModel, tokenName)
}(c.Request.Context())
quotaDelta = quota - preConsumedQuota
for k, v := range resp.Header {
c.Writer.Header().Set(k, v[0])