Fixed the duplicate data check problem, history filter and add tts st…

…ream (#477)
labring · Nov 16, 2023 · fbe1d8c · fbe1d8c
1 parent 1610302
commit fbe1d8c
Show file tree

Hide file tree

Showing 31 changed files with 363 additions and 191 deletions.
diff --git a/packages/global/core/ai/model.d.ts b/packages/global/core/ai/model.d.ts
@@ -29,4 +29,7 @@ export type AudioSpeechModelType = {
   model: string;
   name: string;
   price: number;
+  baseUrl?: string;
+  key?: string;
+  voices: { label: string; value: string; bufferId: string }[];
 };
diff --git a/packages/global/core/ai/model.ts b/packages/global/core/ai/model.ts
@@ -105,11 +105,14 @@ export const defaultAudioSpeechModels: AudioSpeechModelType[] = [
   {
     model: 'tts-1',
     name: 'OpenAI TTS1',
-    price: 0
-  },
-  {
-    model: 'tts-1-hd',
-    name: 'OpenAI TTS1',
-    price: 0
+    price: 0,
+    voices: [
+      { label: 'Alloy', value: 'Alloy', bufferId: 'openai-Alloy' },
+      { label: 'Echo', value: 'Echo', bufferId: 'openai-Echo' },
+      { label: 'Fable', value: 'Fable', bufferId: 'openai-Fable' },
+      { label: 'Onyx', value: 'Onyx', bufferId: 'openai-Onyx' },
+      { label: 'Nova', value: 'Nova', bufferId: 'openai-Nova' },
+      { label: 'Shimmer', value: 'Shimmer', bufferId: 'openai-Shimmer' }
+    ]
   }
 ];
diff --git a/packages/global/core/ai/speech/api.d.ts b/packages/global/core/ai/speech/api.d.ts
diff --git a/packages/global/core/ai/speech/constant.ts b/packages/global/core/ai/speech/constant.ts
diff --git a/packages/global/core/app/type.d.ts b/packages/global/core/app/type.d.ts
@@ -1,7 +1,6 @@
 import { ModuleItemType } from '../module/type';
 import { AppTypeEnum } from './constants';
 import { PermissionTypeEnum } from '../../support/permission/constant';
-import { Text2SpeechVoiceEnum } from '../ai/speech/constant';
 
 export interface AppSchema {
   _id: string;

diff --git a/packages/global/core/chat/type.d.ts b/packages/global/core/chat/type.d.ts
@@ -39,7 +39,6 @@ export type ChatItemSchema = {
   userFeedback?: string;
   adminFeedback?: AdminFbkType;
   [TaskResponseKeyEnum.responseData]?: ChatHistoryItemResType[];
-  tts?: Buffer;
 };
 
 export type AdminFbkType = {
@@ -62,7 +61,7 @@ export type ChatItemType = {
 export type ChatSiteItemType = {
   status: 'loading' | 'running' | 'finish';
   moduleName?: string;
-  ttsBuffer?: Buffer;
+  ttsBuffer?: Uint8Array;
 } & ChatItemType;
 
 export type HistoryItemType = {

diff --git a/packages/service/common/buffer/tts/schema.ts b/packages/service/common/buffer/tts/schema.ts
@@ -0,0 +1,35 @@
+import { connectionMongo, type Model } from '../../../common/mongo';
+const { Schema, model, models } = connectionMongo;
+import { TTSBufferSchemaType } from './type.d';
+
+export const collectionName = 'ttsbuffers';
+
+const TTSBufferSchema = new Schema({
+  bufferId: {
+    type: String,
+    required: true
+  },
+  text: {
+    type: String,
+    required: true
+  },
+  buffer: {
+    type: Buffer,
+    required: true
+  },
+  createTime: {
+    type: Date,
+    default: () => new Date()
+  }
+});
+
+try {
+  TTSBufferSchema.index({ bufferId: 1 });
+  //  24 hour
+  TTSBufferSchema.index({ createTime: 1 }, { expireAfterSeconds: 24 * 60 * 60 });
+} catch (error) {
+  console.log(error);
+}
+
+export const MongoTTSBuffer: Model<TTSBufferSchemaType> =
+  models[collectionName] || model(collectionName, TTSBufferSchema);
diff --git a/packages/service/common/buffer/tts/type.d.ts b/packages/service/common/buffer/tts/type.d.ts
@@ -0,0 +1,5 @@
+export type TTSBufferSchemaType = {
+  bufferId: string;
+  text: string;
+  buffer: Buffer;
+};
diff --git a/packages/service/core/ai/audio/speech.ts b/packages/service/core/ai/audio/speech.ts
@@ -1,26 +1,49 @@
-import { Text2SpeechProps } from '@fastgpt/global/core/ai/speech/api';
+import type { NextApiResponse } from 'next';
 import { getAIApi } from '../config';
 import { defaultAudioSpeechModels } from '../../../../global/core/ai/model';
-import { Text2SpeechVoiceEnum } from '@fastgpt/global/core/ai/speech/constant';
+import { UserModelSchema } from '@fastgpt/global/support/user/type';
 
 export async function text2Speech({
-  model = defaultAudioSpeechModels[0].model,
-  voice = Text2SpeechVoiceEnum.alloy,
+  res,
+  onSuccess,
+  onError,
   input,
-  speed = 1
-}: Text2SpeechProps) {
-  const ai = getAIApi();
-  const mp3 = await ai.audio.speech.create({
+  model = defaultAudioSpeechModels[0].model,
+  voice,
+  speed = 1,
+  props
+}: {
+  res: NextApiResponse;
+  onSuccess: (e: { model: string; buffer: Buffer }) => void;
+  onError: (e: any) => void;
+  input: string;
+  model: string;
+  voice: string;
+  speed?: number;
+  props?: UserModelSchema['openaiAccount'];
+}) {
+  const ai = getAIApi(props);
+  const response = await ai.audio.speech.create({
     model,
+    // @ts-ignore
     voice,
     input,
     response_format: 'mp3',
     speed
   });
-  const buffer = Buffer.from(await mp3.arrayBuffer());
-  return {
-    model,
-    voice,
-    tts: buffer
-  };
+
+  const readableStream = response.body as unknown as NodeJS.ReadableStream;
+  readableStream.pipe(res);
+
+  let bufferStore = Buffer.from([]);
+
+  readableStream.on('data', (chunk) => {
+    bufferStore = Buffer.concat([bufferStore, chunk]);
+  });
+  readableStream.on('end', () => {
+    onSuccess({ model, buffer: bufferStore });
+  });
+  readableStream.on('error', (e) => {
+    onError(e);
+  });
 }
diff --git a/packages/service/core/chat/chatItemSchema.ts b/packages/service/core/chat/chatItemSchema.ts
@@ -68,9 +68,6 @@ const ChatItemSchema = new Schema({
   [TaskResponseKeyEnum.responseData]: {
     type: Array,
     default: []
-  },
-  tts: {
-    type: Buffer
   }
 });
 

diff --git a/projects/app/data/config.json b/projects/app/data/config.json
@@ -102,12 +102,17 @@
     {
       "model": "tts-1",
       "name": "OpenAI TTS1",
-      "price": 0
-    },
-    {
-      "model": "tts-1-hd",
-      "name": "OpenAI TTS1HD",
-      "price": 0
+      "price": 0,
+      "baseUrl": "https://api.openai.com/v1",
+      "key": "",
+      "voices": [
+        { "label": "Alloy", "value": "alloy", "bufferId": "openai-Alloy" },
+        { "label": "Echo", "value": "echo", "bufferId": "openai-Echo" },
+        { "label": "Fable", "value": "fable", "bufferId": "openai-Fable" },
+        { "label": "Onyx", "value": "onyx", "bufferId": "openai-Onyx" },
+        { "label": "Nova", "value": "nova", "bufferId": "openai-Nova" },
+        { "label": "Shimmer", "value": "shimmer", "bufferId": "openai-Shimmer" }
+      ]
     }
   ]
 }
diff --git a/projects/app/public/locales/en/common.json b/projects/app/public/locales/en/common.json
@@ -313,10 +313,12 @@
         "Course": "Document",
         "Delete": "Delete",
         "Index": "Index({{amount}})"
-      }
+      },
+      "input is empty": "The data content cannot be empty"
     },
     "deleteDatasetTips": "Are you sure to delete the knowledge base? Data cannot be recovered after deletion, please confirm!",
     "deleteFolderTips": "Are you sure to delete this folder and all the knowledge bases it contains? Data cannot be recovered after deletion, please confirm!",
+    "import csv tip": "Ensure that the CSV is in UTF-8 format; otherwise, garbled characters will be displayed",
     "recall": {
       "rerank": "Rerank"
     },

diff --git a/projects/app/public/locales/zh/common.json b/projects/app/public/locales/zh/common.json
@@ -313,10 +313,12 @@
         "Course": "说明文档",
         "Delete": "删除数据",
         "Index": "数据索引({{amount}})"
-      }
+      },
+      "input is empty": "数据内容不能为空 "
     },
     "deleteDatasetTips": "确认删除该知识库？删除后数据无法恢复，请确认！",
     "deleteFolderTips": "确认删除该文件夹及其包含的所有知识库？删除后数据无法恢复，请确认！",
+    "import csv tip": "请确保CSV为UTF-8格式，否则会乱码",
     "recall": {
       "rerank": "结果重排"
     },

diff --git a/projects/app/src/components/ChatBox/index.tsx b/projects/app/src/components/ChatBox/index.tsx
@@ -1221,18 +1221,19 @@ function ChatController({
               name={'voice'}
               _hover={{ color: '#E74694' }}
               onClick={async () => {
-                const buffer = await playAudio({
+                const response = await playAudio({
                   buffer: chat.ttsBuffer,
                   chatItemId: chat.dataId,
                   text: chat.value
                 });
-                if (!setChatHistory) return;
+
+                if (!setChatHistory || !response.buffer) return;
                 setChatHistory((state) =>
                   state.map((item) =>
                     item.dataId === chat.dataId
                       ? {
                           ...item,
-                          ttsBuffer: buffer
+                          ttsBuffer: response.buffer
                         }
                       : item
                   )

diff --git a/projects/app/src/global/common/api/systemRes.d.ts b/projects/app/src/global/common/api/systemRes.d.ts
@@ -2,7 +2,8 @@ import type {
   ChatModelItemType,
   FunctionModelItemType,
   LLMModelItemType,
-  VectorModelItemType
+  VectorModelItemType,
+  AudioSpeechModels
 } from '@fastgpt/global/core/ai/model.d';
 
 import type { FeConfigsType } from '@fastgpt/global/common/system/types/index.d';
@@ -12,8 +13,8 @@ export type InitDateResponse = {
   qaModels: LLMModelItemType[];
   cqModels: FunctionModelItemType[];
   extractModels: FunctionModelItemType[];
-  qgModels: LLMModelItemType[];
   vectorModels: VectorModelItemType[];
+  audioSpeechModels: AudioSpeechModels[];
   feConfigs: FeConfigsType;
   priceMd: string;
   systemVersion: string;

diff --git a/projects/app/src/global/core/chat/api.d.ts b/projects/app/src/global/core/chat/api.d.ts
@@ -1,7 +1,6 @@
 import type { AppTTSConfigType } from '@/types/app';
 
 export type GetChatSpeechProps = {
-  chatItemId?: string;
   ttsConfig: AppTTSConfigType;
   input: string;
 };