Add whisper and tts ui (#484)

Co-authored-by: heheer <[email protected]>
labring · Nov 16, 2023 · 4358b6d · 4358b6d
1 parent f6aea48
commit 4358b6d
Show file tree

Hide file tree

Showing 34 changed files with 805 additions and 332 deletions.
diff --git a/docSite/content/docs/development/configuration.md b/docSite/content/docs/development/configuration.md
@@ -123,13 +123,23 @@ weight: 520
     {
       "model": "tts-1",
       "name": "OpenAI TTS1",
-      "price": 0
-    },
-    {
-      "model": "tts-1-hd",
-      "name": "OpenAI TTS1HD",
-      "price": 0
+      "price": 0,
+      "baseUrl": "",
+      "key": "",
+      "voices": [
+        { "label": "Alloy", "value": "alloy", "bufferId": "openai-Alloy" },
+        { "label": "Echo", "value": "echo", "bufferId": "openai-Echo" },
+        { "label": "Fable", "value": "fable", "bufferId": "openai-Fable" },
+        { "label": "Onyx", "value": "onyx", "bufferId": "openai-Onyx" },
+        { "label": "Nova", "value": "nova", "bufferId": "openai-Nova" },
+        { "label": "Shimmer", "value": "shimmer", "bufferId": "openai-Shimmer" }
+      ]
     }
-  ]
+  ],
+  "WhisperModel": {
+    "model": "whisper-1",
+    "name": "Whisper1",
+    "price": 0
+  }
 }
 ```
diff --git a/packages/global/core/ai/model.d.ts b/packages/global/core/ai/model.d.ts
@@ -33,3 +33,9 @@ export type AudioSpeechModelType = {
   key?: string;
   voices: { label: string; value: string; bufferId: string }[];
 };
+
+export type WhisperModelType = {
+  model: string;
+  name: string;
+  price: number;
+};
diff --git a/packages/global/core/ai/model.ts b/packages/global/core/ai/model.ts
@@ -3,7 +3,8 @@ import type {
   ChatModelItemType,
   FunctionModelItemType,
   VectorModelItemType,
-  AudioSpeechModelType
+  AudioSpeechModelType,
+  WhisperModelType
 } from './model.d';
 
 export const defaultChatModels: ChatModelItemType[] = [
@@ -116,3 +117,9 @@ export const defaultAudioSpeechModels: AudioSpeechModelType[] = [
     ]
   }
 ];
+
+export const defaultWhisperModel: WhisperModelType = {
+  model: 'whisper-1',
+  name: 'Whisper1',
+  price: 0
+};
diff --git a/packages/service/common/file/upload/multer.ts b/packages/service/common/file/upload/multer.ts
@@ -32,10 +32,10 @@ export function getUploadModel({ maxSize = 500 }: { maxSize?: number }) {
       })
     }).any();
 
-    async doUpload(req: NextApiRequest, res: NextApiResponse) {
+    async doUpload<T = Record<string, any>>(req: NextApiRequest, res: NextApiResponse) {
       return new Promise<{
         files: FileType[];
-        metadata: Record<string, any>;
+        metadata: T;
         bucketName?: `${BucketNameEnum}`;
       }>((resolve, reject) => {
         // @ts-ignore

diff --git a/packages/service/core/ai/config.ts b/packages/service/core/ai/config.ts
@@ -6,7 +6,7 @@ export const baseUrl = process.env.ONEAPI_URL || openaiBaseUrl;
 
 export const systemAIChatKey = process.env.CHAT_API_KEY || '';
 
-export const getAIApi = (props?: UserModelSchema['openaiAccount'], timeout = 6000) => {
+export const getAIApi = (props?: UserModelSchema['openaiAccount'], timeout = 60000) => {
   return new OpenAI({
     apiKey: props?.key || systemAIChatKey,
     baseURL: props?.baseUrl || baseUrl,

diff --git a/projects/app/data/config.json b/projects/app/data/config.json
@@ -103,7 +103,7 @@
       "model": "tts-1",
       "name": "OpenAI TTS1",
       "price": 0,
-      "baseUrl": "https://api.openai.com/v1",
+      "baseUrl": "",
       "key": "",
       "voices": [
         { "label": "Alloy", "value": "alloy", "bufferId": "openai-Alloy" },
@@ -114,5 +114,10 @@
         { "label": "Shimmer", "value": "shimmer", "bufferId": "openai-Shimmer" }
       ]
     }
-  ]
+  ],
+  "WhisperModel": {
+    "model": "whisper-1",
+    "name": "Whisper1",
+    "price": 0
+  }
 }
diff --git a/projects/app/package.json b/projects/app/package.json
@@ -1,6 +1,6 @@
 {
   "name": "app",
-  "version": "4.6",
+  "version": "4.6.1",
   "private": false,
   "scripts": {
     "dev": "next dev",

diff --git a/projects/app/public/locales/en/common.json b/projects/app/public/locales/en/common.json
@@ -73,6 +73,7 @@
     "Complete Response": "Complete Response",
     "Confirm to clear history": "Confirm to clear history?",
     "Confirm to clear share chat history": " Are you sure to delete all chats?",
+    "Converting to text": "Converting to text...",
     "Exit Chat": "Exit",
     "Feedback Close": "Close Feedback",
     "Feedback Failed": "Feedback Failed",
@@ -216,23 +217,31 @@
     "app": {
       "Next Step Guide": "Next step guide",
       "Question Guide Tip": "At the end of the conversation, three leading questions will be asked.",
+      "Select TTS": "Select TTS",
       "TTS": "Audio Speech",
       "TTS Tip": "After this function is enabled, the voice playback function can be used after each conversation. Use of this feature may incur additional charges.",
       "tts": {
         "Close": "NoUse",
         "Model alloy": "Female - Alloy",
         "Model echo": "Male - Echo",
+        "Speech model": "Speech model",
+        "Speech speed": "Speed",
         "Test Listen": "Test",
         "Test Listen Text": "Hello, this is FastGPT, how can I help you?",
         "Web": "Browser (free)"
       }
     },
     "chat": {
       "Audio Speech Error": "Audio Speech Error",
+      "Speaking": "I'm listening...",
       "Record": "Speech",
       "Restart": "Restart",
       "Send Message": "Send Message",
-      "Stop Speak": "Stop Speak"
+      "Stop Speak": "Stop Speak",
+      "Type a message": "Input problem",
+      "tts": {
+        "Stop Speech": "Stop"
+      }
     },
     "dataset": {
       "Choose Dataset": "Choose Dataset",
@@ -580,7 +589,8 @@
   "wallet": {
     "bill": {
       "Audio Speech": "Audio Speech",
-      "bill username": "User"
+      "bill username": "User",
+      "Whisper": "Whisper"
     }
   }
 }
diff --git a/projects/app/public/locales/zh/common.json b/projects/app/public/locales/zh/common.json
@@ -73,6 +73,7 @@
     "Complete Response": "完整响应",
     "Confirm to clear history": "确认清空该应用的在线聊天记录？分享和 API 调用的记录不会被清空。",
     "Confirm to clear share chat history": "确认删除所有聊天记录？",
+    "Converting to text": "正在转换为文本...",
     "Exit Chat": "退出聊天",
     "Feedback Close": "关闭反馈",
     "Feedback Failed": "提交反馈异常",
@@ -216,12 +217,15 @@
     "app": {
       "Next Step Guide": "下一步指引",
       "Question Guide Tip": "对话结束后，会为生成 3 个引导性问题。",
+      "Select TTS": "选择语音播放模式",
       "TTS": "语音播报",
       "TTS Tip": "开启后，每次对话后可使用语音播放功能。使用该功能可能产生额外费用。",
       "tts": {
         "Close": "不使用",
         "Model alloy": "女声 - Alloy",
         "Model echo": "男声 - Echo",
+        "Speech model": "语音模型",
+        "Speech speed": "语速",
         "Test Listen": "试听",
         "Test Listen Text": "你好，我是 FastGPT，有什么可以帮助你么？",
         "Web": "浏览器自带(免费)"
@@ -232,7 +236,12 @@
       "Record": "语音输入",
       "Restart": "重开对话",
       "Send Message": "发送",
-      "Stop Speak": "停止录音"
+      "Speaking": "我在听，请说...",
+      "Stop Speak": "停止录音",
+      "Type a message": "输入问题",
+      "tts": {
+        "Stop Speech": "停止"
+      }
     },
     "dataset": {
       "Choose Dataset": "关联知识库",
@@ -580,7 +589,8 @@
   "wallet": {
     "bill": {
       "Audio Speech": "语音播报",
-      "bill username": "用户"
+      "bill username": "用户",
+      "Whisper": "语音输入"
     }
   }
 }