feat: Support Ali TTS

common/model-ratio.go

@@ -90,6 +90,49 @@ var ModelRatio = map[string]float64{
 	"qwen-turbo":                0.8572, // ￥0.012 / 1k tokens
 	"qwen-plus":                 10,     // ￥0.14 / 1k tokens
 	"text-embedding-v1":         0.05,   // ￥0.0007 / 1k tokens
+	"sambert-zhichu-v1":         0.777,  // todo
+	"sambert-zhiwei-v1":         0.777,  // todo
+	"sambert-zhixiang-v1":       0.777,  // todo
+	"sambert-zhide-v1":          0.777,  // todo
+	"sambert-zhijia-v1":         0.777,  // todo
+	"sambert-zhinan-v1":         0.777,  // todo
+	"sambert-zhiqi-v1":          0.777,  // todo
+	"sambert-zhiqian-v1":        0.777,  // todo
+	"sambert-zhiru-v1":          0.777,  // todo
+	"sambert-zhimiao-emo-v1":    0.777,  // todo
+	"sambert-zhida-v1":          0.777,  // todo
+	"sambert-zhifei-v1":         0.777,  // todo
+	"sambert-zhigui-v1":         0.777,  // todo
+	"sambert-zhihao-v1":         0.777,  // todo
+	"sambert-zhijing-v1":        0.777,  // todo
+	"sambert-zhilun-v1":         0.777,  // todo
+	"sambert-zhimao-v1":         0.777,  // todo
+	"sambert-zhiming-v1":        0.777,  // todo
+	"sambert-zhimo-v1":          0.777,  // todo
+	"sambert-zhina-v1":          0.777,  // todo
+	"sambert-zhishu-v1":         0.777,  // todo
+	"sambert-zhishuo-v1":        0.777,  // todo
+	"sambert-zhistella-v1":      0.777,  // todo
+	"sambert-zhiting-v1":        0.777,  // todo
+	"sambert-zhixiao-v1":        0.777,  // todo
+	"sambert-zhiya-v1":          0.777,  // todo
+	"sambert-zhiye-v1":          0.777,  // todo
+	"sambert-zhiying-v1":        0.777,  // todo
+	"sambert-zhiyuan-v1":        0.777,  // todo
+	"sambert-zhiyue-v1":         0.777,  // todo
+	"sambert-camila-v1":         0.777,  // todo
+	"sambert-perla-v1":          0.777,  // todo
+	"sambert-indah-v1":          0.777,  // todo
+	"sambert-clara-v1":          0.777,  // todo
+	"sambert-hanna-v1":          0.777,  // todo
+	"sambert-beth-v1":           0.777,  // todo
+	"sambert-betty-v1":          0.777,  // todo
+	"sambert-cally-v1":          0.777,  // todo
+	"sambert-cindy-v1":          0.777,  // todo
+	"sambert-eva-v1":            0.777,  // todo
+	"sambert-donna-v1":          0.777,  // todo
+	"sambert-brian-v1":          0.777,  // todo
+	"sambert-waan-v1":           0.777,  // todo
 	"SparkDesk":                 1.2858, // ￥0.018 / 1k tokens
 	"360GPT_S2_V9":              0.8572, // ¥0.012 / 1k tokens
 	"embedding-bert-512-v1":     0.0715, // ¥0.001 / 1k tokens

controller/model.go

@@ -486,6 +486,385 @@ func init() {
 			Root:       "text-embedding-v1",
 			Parent:     nil,
 		},
+		{
+			Id:         "sambert-zhichu-v1",
+			Object:     "model",
+			Created:    1702621721,
+			OwnedBy:    "ali",
+			Permission: permission,
+			Root:       "sambert-zhichu-v1",
+			Parent:     nil,
+		},
+		{
+			Id:         "sambert-zhiwei-v1",
+			Object:     "model",
+			Created:    1702621721,
+			OwnedBy:    "ali",
+			Permission: permission,
+			Root:       "sambert-zhiwei-v1",
+			Parent:     nil,
+		},
+		{
+			Id:         "sambert-zhixiang-v1",
+			Object:     "model",
+			Created:    1702621721,
+			OwnedBy:    "ali",
+			Permission: permission,
+			Root:       "sambert-zhixiang-v1",
+			Parent:     nil,
+		},
+		{
+			Id:         "sambert-zhide-v1",
+			Object:     "model",
+			Created:    1702621721,
+			OwnedBy:    "ali",
+			Permission: permission,
+			Root:       "sambert-zhide-v1",
+			Parent:     nil,
+		},
+		{
+			Id:         "sambert-zhijia-v1",
+			Object:     "model",
+			Created:    1702621721,
+			OwnedBy:    "ali",
+			Permission: permission,
+			Root:       "sambert-zhijia-v1",
+			Parent:     nil,
+		},
+		{
+			Id:         "sambert-zhinan-v1",
+			Object:     "model",
+			Created:    1702621721,
+			OwnedBy:    "ali",
+			Permission: permission,
+			Root:       "sambert-zhinan-v1",
+			Parent:     nil,
+		},
+		{
+			Id:         "sambert-zhiqi-v1",
+			Object:     "model",
+			Created:    1702621721,
+			OwnedBy:    "ali",
+			Permission: permission,
+			Root:       "sambert-zhiqi-v1",
+			Parent:     nil,
+		},
+		{
+			Id:         "sambert-zhiqian-v1",
+			Object:     "model",
+			Created:    1702621721,
+			OwnedBy:    "ali",
+			Permission: permission,
+			Root:       "sambert-zhiqian-v1",
+			Parent:     nil,
+		},
+		{
+			Id:         "sambert-zhiru-v1",
+			Object:     "model",
+			Created:    1702621721,
+			OwnedBy:    "ali",
+			Permission: permission,
+			Root:       "sambert-zhiru-v1",
+			Parent:     nil,
+		},
+		{
+			Id:         "sambert-zhimiao-emo-v1",
+			Object:     "model",
+			Created:    1702621721,
+			OwnedBy:    "ali",
+			Permission: permission,
+			Root:       "sambert-zhimiao-emo-v1",
+			Parent:     nil,
+		},
+		{
+			Id:         "sambert-zhida-v1",
+			Object:     "model",
+			Created:    1702621721,
+			OwnedBy:    "ali",
+			Permission: permission,
+			Root:       "sambert-zhida-v1",
+			Parent:     nil,
+		},
+		{
+			Id:         "sambert-zhifei-v1",
+			Object:     "model",
+			Created:    1702621721,
+			OwnedBy:    "ali",
+			Permission: permission,
+			Root:       "sambert-zhifei-v1",
+			Parent:     nil,
+		},
+		{
+			Id:         "sambert-zhigui-v1",
+			Object:     "model",
+			Created:    1702621721,
+			OwnedBy:    "ali",
+			Permission: permission,
+			Root:       "sambert-zhigui-v1",
+			Parent:     nil,
+		},
+		{
+			Id:         "sambert-zhihao-v1",
+			Object:     "model",
+			Created:    1702621721,
+			OwnedBy:    "ali",
+			Permission: permission,
+			Root:       "sambert-zhihao-v1",
+			Parent:     nil,
+		},
+		{
+			Id:         "sambert-zhijing-v1",
+			Object:     "model",
+			Created:    1702621721,
+			OwnedBy:    "ali",
+			Permission: permission,
+			Root:       "sambert-zhijing-v1",
+			Parent:     nil,
+		},
+		{
+			Id:         "sambert-zhilun-v1",
+			Object:     "model",
+			Created:    1702621721,
+			OwnedBy:    "ali",
+			Permission: permission,
+			Root:       "sambert-zhilun-v1",
+			Parent:     nil,
+		},
+		{
+			Id:         "sambert-zhimao-v1",
+			Object:     "model",
+			Created:    1702621721,
+			OwnedBy:    "ali",
+			Permission: permission,
+			Root:       "sambert-zhimao-v1",
+			Parent:     nil,
+		},
+		{
+			Id:         "sambert-zhiming-v1",
+			Object:     "model",
+			Created:    1702621721,
+			OwnedBy:    "ali",
+			Permission: permission,
+			Root:       "sambert-zhiming-v1",
+			Parent:     nil,
+		},
+		{
+			Id:         "sambert-zhimo-v1",
+			Object:     "model",
+			Created:    1702621721,
+			OwnedBy:    "ali",
+			Permission: permission,
+			Root:       "sambert-zhimo-v1",
+			Parent:     nil,
+		},
+		{
+			Id:         "sambert-zhina-v1",
+			Object:     "model",
+			Created:    1702621721,
+			OwnedBy:    "ali",
+			Permission: permission,
+			Root:       "sambert-zhina-v1",
+			Parent:     nil,
+		},
+		{
+			Id:         "sambert-zhishu-v1",
+			Object:     "model",
+			Created:    1702621721,
+			OwnedBy:    "ali",
+			Permission: permission,
+			Root:       "sambert-zhishu-v1",
+			Parent:     nil,
+		},
+		{
+			Id:         "sambert-zhishuo-v1",
+			Object:     "model",
+			Created:    1702621721,
+			OwnedBy:    "ali",
+			Permission: permission,
+			Root:       "sambert-zhishuo-v1",
+			Parent:     nil,
+		},
+		{
+			Id:         "sambert-zhistella-v1",
+			Object:     "model",
+			Created:    1702621721,
+			OwnedBy:    "ali",
+			Permission: permission,
+			Root:       "sambert-zhistella-v1",
+			Parent:     nil,
+		},
+		{
+			Id:         "sambert-zhiting-v1",
+			Object:     "model",
+			Created:    1702621721,
+			OwnedBy:    "ali",
+			Permission: permission,
+			Root:       "sambert-zhiting-v1",
+			Parent:     nil,
+		},
+		{
+			Id:         "sambert-zhixiao-v1",
+			Object:     "model",
+			Created:    1702621721,
+			OwnedBy:    "ali",
+			Permission: permission,
+			Root:       "sambert-zhixiao-v1",
+			Parent:     nil,
+		},
+		{
+			Id:         "sambert-zhiya-v1",
+			Object:     "model",
+			Created:    1702621721,
+			OwnedBy:    "ali",
+			Permission: permission,
+			Root:       "sambert-zhiya-v1",
+			Parent:     nil,
+		},
+		{
+			Id:         "sambert-zhiye-v1",
+			Object:     "model",
+			Created:    1702621721,
+			OwnedBy:    "ali",
+			Permission: permission,
+			Root:       "sambert-zhiye-v1",
+			Parent:     nil,
+		},
+		{
+			Id:         "sambert-zhiying-v1",
+			Object:     "model",
+			Created:    1702621721,
+			OwnedBy:    "ali",
+			Permission: permission,
+			Root:       "sambert-zhiying-v1",
+			Parent:     nil,
+		},
+		{
+			Id:         "sambert-zhiyuan-v1",
+			Object:     "model",
+			Created:    1702621721,
+			OwnedBy:    "ali",
+			Permission: permission,
+			Root:       "sambert-zhiyuan-v1",
+			Parent:     nil,
+		},
+		{
+			Id:         "sambert-zhiyue-v1",
+			Object:     "model",
+			Created:    1702621721,
+			OwnedBy:    "ali",
+			Permission: permission,
+			Root:       "sambert-zhiyue-v1",
+			Parent:     nil,
+		},
+		{
+			Id:         "sambert-camila-v1",
+			Object:     "model",
+			Created:    1702621721,
+			OwnedBy:    "ali",
+			Permission: permission,
+			Root:       "sambert-camila-v1",
+			Parent:     nil,
+		},
+		{
+			Id:         "sambert-perla-v1",
+			Object:     "model",
+			Created:    1702621721,
+			OwnedBy:    "ali",
+			Permission: permission,
+			Root:       "sambert-perla-v1",
+			Parent:     nil,
+		},
+		{
+			Id:         "sambert-indah-v1",
+			Object:     "model",
+			Created:    1702621721,
+			OwnedBy:    "ali",
+			Permission: permission,
+			Root:       "sambert-indah-v1",
+			Parent:     nil,
+		},
+		{
+			Id:         "sambert-clara-v1",
+			Object:     "model",
+			Created:    1702621721,
+			OwnedBy:    "ali",
+			Permission: permission,
+			Root:       "sambert-clara-v1",
+			Parent:     nil,
+		},
+		{
+			Id:         "sambert-hanna-v1",
+			Object:     "model",
+			Created:    1702621721,
+			OwnedBy:    "ali",
+			Permission: permission,
+			Root:       "sambert-hanna-v1",
+			Parent:     nil,
+		},
+		{
+			Id:         "sambert-beth-v1",
+			Object:     "model",
+			Created:    1702621721,
+			OwnedBy:    "ali",
+			Permission: permission,
+			Root:       "sambert-beth-v1",
+			Parent:     nil,
+		},
+		{
+			Id:         "sambert-cally-v1",
+			Object:     "model",
+			Created:    1702621721,
+			OwnedBy:    "ali",
+			Permission: permission,
+			Root:       "sambert-cally-v1",
+			Parent:     nil,
+		},
+		{
+			Id:         "sambert-cindy-v1",
+			Object:     "model",
+			Created:    1702621721,
+			OwnedBy:    "ali",
+			Permission: permission,
+			Root:       "sambert-cindy-v1",
+			Parent:     nil,
+		},
+		{
+			Id:         "sambert-eva-v1",
+			Object:     "model",
+			Created:    1702621721,
+			OwnedBy:    "ali",
+			Permission: permission,
+			Root:       "sambert-eva-v1",
+			Parent:     nil,
+		},
+		{
+			Id:         "sambert-donna-v1",
+			Object:     "model",
+			Created:    1702621721,
+			OwnedBy:    "ali",
+			Permission: permission,
+			Root:       "sambert-donna-v1",
+			Parent:     nil,
+		},
+		{
+			Id:         "sambert-brian-v1",
+			Object:     "model",
+			Created:    1702621721,
+			OwnedBy:    "ali",
+			Permission: permission,
+			Root:       "sambert-brian-v1",
+			Parent:     nil,
+		},
+		{
+			Id:         "sambert-waan-v1",
+			Object:     "model",
+			Created:    1702621721,
+			OwnedBy:    "ali",
+			Permission: permission,
+			Root:       "sambert-waan-v1",
+			Parent:     nil,
+		},
+
 		{
 			Id:         "SparkDesk",
 			Object:     "model",

controller/relay-ali.go

@@ -4,6 +4,8 @@ import (
 	"bufio"
 	"encoding/json"
 	"github.com/gin-gonic/gin"
+	"github.com/google/uuid"
+	"github.com/gorilla/websocket"
 	"io"
 	"net/http"
 	"one-api/common"
@@ -35,6 +37,61 @@ type AliChatRequest struct {
 	Parameters AliParameters `json:"parameters,omitempty"`
 }
 
+type AliTaskResponse struct {
+	StatusCode int    `json:"status_code,omitempty"`
+	RequestId  string `json:"request_id,omitempty"`
+	Code       string `json:"code,omitempty"`
+	Message    string `json:"message,omitempty"`
+	Output     struct {
+		TaskId     string `json:"task_id,omitempty"`
+		TaskStatus string `json:"task_status,omitempty"`
+		Code       string `json:"code,omitempty"`
+		Message    string `json:"message,omitempty"`
+		Results    []struct {
+			B64Image string `json:"b64_image,omitempty"`
+			Url      string `json:"url,omitempty"`
+			Code     string `json:"code,omitempty"`
+			Message  string `json:"message,omitempty"`
+		} `json:"results,omitempty"`
+		TaskMetrics struct {
+			Total     int `json:"TOTAL,omitempty"`
+			Succeeded int `json:"SUCCEEDED,omitempty"`
+			Failed    int `json:"FAILED,omitempty"`
+		} `json:"task_metrics,omitempty"`
+	} `json:"output,omitempty"`
+	Usage Usage `json:"usage"`
+}
+
+type AliHeader struct {
+	Action    string `json:"action,omitempty"`
+	Streaming string `json:"streaming,omitempty"`
+	TaskID    string `json:"task_id,omitempty"`
+	Event     string `json:"event,omitempty"`
+}
+
+type AliPayload struct {
+	Model      string `json:"model,omitempty"`
+	Task       string `json:"task,omitempty"`
+	TaskGroup  string `json:"task_group,omitempty"`
+	Function   string `json:"function,omitempty"`
+	Parameters struct {
+		SampleRate int     `json:"sample_rate,omitempty"`
+		Rate       float64 `json:"rate,omitempty"`
+		Format     string  `json:"format,omitempty"`
+	} `json:"parameters,omitempty"`
+	Input struct {
+		Text string `json:"text,omitempty"`
+	} `json:"input,omitempty"`
+	Usage struct {
+		Characters int `json:"characters,omitempty"`
+	} `json:"usage,omitempty"`
+}
+
+type AliWSSMessage struct {
+	Header  AliHeader  `json:"header,omitempty"`
+	Payload AliPayload `json:"payload,omitempty"`
+}
+
 type AliEmbeddingRequest struct {
 	Model string `json:"model"`
 	Input struct {
@@ -119,6 +176,23 @@ func requestOpenAI2Ali(request GeneralOpenAIRequest) *AliChatRequest {
 	}
 }
 
+func requestOpenAI2AliTTS(request TextToSpeechRequest) *AliWSSMessage {
+	var ttsRequest AliWSSMessage
+	ttsRequest.Header.Action = "run-task"
+	ttsRequest.Header.Streaming = "out"
+	ttsRequest.Header.TaskID = uuid.New().String()
+	ttsRequest.Payload.Function = "SpeechSynthesizer"
+	ttsRequest.Payload.Input.Text = request.Input
+	ttsRequest.Payload.Model = request.Model
+	ttsRequest.Payload.Parameters.Format = request.ResponseFormat
+	//ttsRequest.Payload.Parameters.SampleRate = 48000
+	ttsRequest.Payload.Parameters.Rate = request.Speed
+	ttsRequest.Payload.Task = "tts"
+	ttsRequest.Payload.TaskGroup = "audio"
+
+	return &ttsRequest
+}
+
 func embeddingRequestOpenAI2Ali(request GeneralOpenAIRequest) *AliEmbeddingRequest {
 	return &AliEmbeddingRequest{
 		Model: "text-embedding-v1",
@@ -327,3 +401,57 @@ func aliHandler(c *gin.Context, resp *http.Response) (*OpenAIErrorWithStatusCode
 	_, err = c.Writer.Write(jsonResponse)
 	return nil, &fullTextResponse.Usage
 }
+
+func aliTTSHandler(c *gin.Context, req TextToSpeechRequest) (*OpenAIErrorWithStatusCode, *Usage) {
+	Authorization := c.Request.Header.Get("Authorization")
+	baseURL := "wss://dashscope.aliyuncs.com/api-ws/v1/inference"
+	var usage Usage
+
+	conn, _, err := websocket.DefaultDialer.Dial(baseURL, http.Header{"Authorization": {Authorization}})
+	if err != nil {
+		return errorWrapper(err, "wss_conn_failed", http.StatusInternalServerError), nil
+	}
+	defer conn.Close()
+
+	message := requestOpenAI2AliTTS(req)
+
+	if err := conn.WriteJSON(message); err != nil {
+		return errorWrapper(err, "wss_write_msg_failed", http.StatusInternalServerError), nil
+	}
+
+	const chunkSize = 1024
+
+	for {
+		messageType, audioData, err := conn.ReadMessage()
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			return errorWrapper(err, "wss_read_msg_failed", http.StatusInternalServerError), nil
+		}
+
+		var msg AliWSSMessage
+		switch messageType {
+		case websocket.TextMessage:
+			err = json.Unmarshal(audioData, &msg)
+			if msg.Header.Event == "task-finished" {
+				usage.TotalTokens = msg.Payload.Usage.Characters
+				return nil, &usage
+			}
+		case websocket.BinaryMessage:
+			for i := 0; i < len(audioData); i += chunkSize {
+				end := i + chunkSize
+				if end > len(audioData) {
+					end = len(audioData)
+				}
+				chunk := audioData[i:end]
+				_, writeErr := c.Writer.Write(chunk)
+				if writeErr != nil {
+					return errorWrapper(writeErr, "write_audio_failed", http.StatusInternalServerError), nil
+				}
+			}
+		}
+	}
+
+	return nil, &usage
+}

controller/relay-audio.go

@@ -90,6 +90,12 @@ func relayAudioHelper(c *gin.Context, relayMode int) *OpenAIErrorWithStatusCode
 		}
 	}
 
+	apiType := APITypeOpenAI
+	switch channelType {
+	case common.ChannelTypeAli:
+		apiType = APITypeAli
+	}
+
 	baseURL := common.ChannelBaseURLs[channelType]
 	requestURL := c.Request.URL.String()
 	if c.GetString("base_url") != "" {
@@ -103,11 +109,33 @@ func relayAudioHelper(c *gin.Context, relayMode int) *OpenAIErrorWithStatusCode
 		fullRequestURL = fmt.Sprintf("%s/openai/deployments/%s/audio/transcriptions?api-version=%s", baseURL, audioModel, apiVersion)
 	}
 
+	quotaDelta := 0
+	defer func(ctx context.Context) {
+		go postConsumeQuota(ctx, tokenId, quotaDelta, quota, userId, channelId, modelRatio, groupRatio, audioModel, tokenName)
+	}(c.Request.Context())
+
 	requestBody := &bytes.Buffer{}
 	_, err = io.Copy(requestBody, c.Request.Body)
 	if err != nil {
 		return errorWrapper(err, "new_request_body_failed", http.StatusInternalServerError)
 	}
+
+	switch apiType {
+	case APITypeAli:
+		if relayMode == RelayModeAudioSpeech {
+			err, usage := aliTTSHandler(c, ttsRequest)
+			if err != nil {
+				return err
+			}
+
+			if usage != nil {
+				quota = usage.TotalTokens
+			}
+
+			return nil
+		}
+	}
+
 	c.Request.Body = io.NopCloser(bytes.NewBuffer(requestBody.Bytes()))
 	responseFormat := c.DefaultPostForm("response_format", "json")
 
@@ -195,10 +223,8 @@ func relayAudioHelper(c *gin.Context, relayMode int) *OpenAIErrorWithStatusCode
 		}
 		return relayErrorHandler(resp)
 	}
-	quotaDelta := quota - preConsumedQuota
+
-	defer func(ctx context.Context) {
+	quotaDelta = quota - preConsumedQuota
-		go postConsumeQuota(ctx, tokenId, quotaDelta, quota, userId, channelId, modelRatio, groupRatio, audioModel, tokenName)
-	}(c.Request.Context())
 
 	for k, v := range resp.Header {
 		c.Writer.Header().Set(k, v[0])