4.6.2-alpha (#517)

labring · Nov 25, 2023 · 3acbf1a · 3acbf1a
1 parent 9cb4280
commit 3acbf1a
Show file tree

Hide file tree

Showing 39 changed files with 613 additions and 179 deletions.
diff --git a/packages/global/common/string/tools.ts b/packages/global/common/string/tools.ts
@@ -13,7 +13,8 @@ export const hashStr = (str: string) => {
 };
 
 /* simple text, remove chinese space and extra \n */
-export const simpleText = (text: string) => {
+export const simpleText = (text = '') => {
+  text = text.trim();
   text = text.replace(/([\u4e00-\u9fa5])[\s&&[^\n]]+([\u4e00-\u9fa5])/g, '$1$2');
   text = text.replace(/\r\n|\r/g, '\n');
   text = text.replace(/\n{3,}/g, '\n\n');

diff --git a/packages/global/core/dataset/type.d.ts b/packages/global/core/dataset/type.d.ts
@@ -59,6 +59,7 @@ export type DatasetDataSchemaType = {
   collectionId: string;
   q: string; // large chunks or question
   a: string; // answer or custom content
+  fullTextToken: string;
   indexes: DatasetDataIndexItemType[];
 };
 
@@ -83,6 +84,9 @@ export type DatasetTrainingSchemaType = {
 export type CollectionWithDatasetType = Omit<DatasetCollectionSchemaType, 'datasetId'> & {
   datasetId: DatasetSchemaType;
 };
+export type DatasetDataWithCollectionType = Omit<DatasetDataSchemaType, 'collectionId'> & {
+  collectionId: DatasetCollectionSchemaType;
+};
 
 /* ================= dataset ===================== */
 export type DatasetItemType = Omit<DatasetSchemaType, 'vectorModel' | 'agentModel'> & {
@@ -130,6 +134,6 @@ export type DatasetFileSchema = {
 };
 
 /* ============= search =============== */
-export type SearchDataResponseItemType = DatasetDataItemType & {
+export type SearchDataResponseItemType = Omit<DatasetDataItemType, 'isOwner' | 'canWrite'> & {
   score: number;
 };
diff --git a/packages/global/core/module/node/type.d.ts b/packages/global/core/module/node/type.d.ts
@@ -20,7 +20,7 @@ export type FlowNodeChangeProps = {
 };
 
 export type FlowNodeInputItemType = {
-  key: `${ModuleInputKeyEnum}`;
+  key: `${ModuleInputKeyEnum}` | string;
   type: `${FlowNodeInputTypeEnum}`; // Decide on a render style
   value?: any;
   valueType?: `${ModuleDataTypeEnum}`; // data type

diff --git a/packages/global/core/module/type.d.ts b/packages/global/core/module/type.d.ts
@@ -25,7 +25,7 @@ export type moduleTemplateListType = {
 // store module type
 export type ModuleItemType = {
   name: string;
-  logo?: string;
+  avatar?: string;
   intro?: string;
   moduleId: string;
   position?: {

diff --git a/packages/service/core/dataset/data/schema.ts b/packages/service/core/dataset/data/schema.ts
@@ -43,13 +43,9 @@ const DatasetDataSchema = new Schema({
     type: String,
     default: ''
   },
-  qToken: {
+  fullTextToken: {
     type: String,
-    default: ''
-  },
-  aToken: {
-    type: String,
-    default: ''
+    required: true
   },
   indexes: {
     type: [
@@ -82,7 +78,7 @@ try {
   DatasetDataSchema.index({ datasetId: 1 });
   DatasetDataSchema.index({ collectionId: 1 });
   // full text index
-  DatasetDataSchema.index({ qToken: 'text', aToken: 'text' });
+  DatasetDataSchema.index({ fullTextToken: 'text' });
 } catch (error) {
   console.log(error);
 }

diff --git a/packages/service/package.json b/packages/service/package.json
@@ -4,23 +4,23 @@
   "dependencies": {
     "@fastgpt/global": "workspace:*",
     "axios": "^1.5.1",
-    "nextjs-cors": "^2.1.2",
-    "next": "13.5.2",
     "cookie": "^0.5.0",
+    "encoding": "^0.1.13",
     "jsonwebtoken": "^9.0.2",
     "mongoose": "^7.0.2",
-    "winston": "^3.10.0",
-    "winston-mongodb": "^5.1.1",
-    "tunnel": "^0.0.6",
-    "encoding": "^0.1.13",
+    "nanoid": "^4.0.1",
+    "next": "13.5.2",
+    "nextjs-cors": "^2.1.2",
     "pg": "^8.10.0",
-    "nanoid": "^4.0.1"
+    "tunnel": "^0.0.6",
+    "winston": "^3.10.0",
+    "winston-mongodb": "^5.1.1"
   },
   "devDependencies": {
-    "@types/tunnel": "^0.0.4",
-    "@types/pg": "^8.6.6",
-    "@types/node": "^20.8.5",
     "@types/cookie": "^0.5.2",
-    "@types/jsonwebtoken": "^9.0.3"
+    "@types/jsonwebtoken": "^9.0.3",
+    "@types/node": "^20.8.5",
+    "@types/pg": "^8.6.6",
+    "@types/tunnel": "^0.0.4"
   }
 }
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
diff --git a/projects/app/package.json b/projects/app/package.json
@@ -20,6 +20,7 @@
     "@fastgpt/service": "workspace:*",
     "@fastgpt/web": "workspace:*",
     "@mozilla/readability": "^0.4.4",
+    "@node-rs/jieba": "^1.7.2",
     "@tanstack/react-query": "^4.24.10",
     "@types/nprogress": "^0.2.0",
     "axios": "^1.5.1",

diff --git a/projects/app/public/locales/en/common.json b/projects/app/public/locales/en/common.json
@@ -261,6 +261,9 @@
       "data": {
         "Edit": "Edit Data",
         "id": "Data ID"
+      },
+      "test": {
+        "Test Result": "Results"
       }
     },
     "module": {

diff --git a/projects/app/public/locales/zh/common.json b/projects/app/public/locales/zh/common.json
@@ -261,6 +261,9 @@
       "data": {
         "Edit": "编辑数据",
         "id": "数据ID"
+      },
+      "test": {
+        "Test Result": "测试结果"
       }
     },
     "module": {

diff --git a/projects/app/src/components/ChatBox/MessageInput.tsx b/projects/app/src/components/ChatBox/MessageInput.tsx
@@ -342,7 +342,7 @@ ${images.map((img) => JSON.stringify({ src: img.src })).join('\n')}
                 const items = clipboardData.items;
                 const files = Array.from(items)
                   .map((item) => (item.kind === 'file' ? item.getAsFile() : undefined))
-                  .filter((item) => item) as File[];
+                  .filter(Boolean) as File[];
                 onSelectFile(files);
               }
             }}

diff --git a/projects/app/src/components/ChatBox/ResponseTags.tsx b/projects/app/src/components/ChatBox/ResponseTags.tsx
@@ -40,7 +40,7 @@ const ResponseTags = ({ responseData = [] }: { responseData?: ChatHistoryItemRes
       .filter((item) => item.moduleType === FlowNodeTypeEnum.chatNode)
       .map((item) => item.quoteList)
       .flat()
-      .filter((item) => item) as SearchDataResponseItemType[];
+      .filter(Boolean) as SearchDataResponseItemType[];
     const sourceList = quoteList.reduce(
       (acc: Record<string, SearchDataResponseItemType[]>, cur) => {
         if (!acc[cur.sourceName]) {

diff --git a/projects/app/src/global/core/api/datasetReq.d.ts b/projects/app/src/global/core/api/datasetReq.d.ts
@@ -18,13 +18,6 @@ export type DatasetUpdateParams = {
   agentModel?: LLMModelItemType;
 };
 
-export type SearchTestProps = {
-  datasetId: string;
-  text: string;
-  limit?: number;
-  rerank?: boolean;
-};
-
 /* ======= collections =========== */
 export type GetDatasetCollectionsProps = RequestPaging & {
   datasetId: string;

diff --git a/projects/app/src/global/core/dataset/api.d.ts b/projects/app/src/global/core/dataset/api.d.ts
@@ -1,6 +1,9 @@
 import { PushDatasetDataChunkProps } from '@fastgpt/global/core/dataset/api';
 import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constant';
-import { DatasetDataIndexItemType } from '@fastgpt/global/core/dataset/type';
+import {
+  DatasetDataIndexItemType,
+  SearchDataResponseItemType
+} from '@fastgpt/global/core/dataset/type';
 
 /* ================= dataset ===================== */
 export type CreateDatasetParams = {
@@ -34,3 +37,15 @@ export type UpdateDatasetDataProps = {
     dataId?: string; // pg data id
   })[];
 };
+
+/* -------------- search ---------------- */
+export type SearchTestProps = {
+  datasetId: string;
+  text: string;
+  limit?: number;
+  rerank?: boolean;
+};
+export type SearchTestResponse = {
+  list: SearchDataResponseItemType[];
+  duration: string;
+};
diff --git a/projects/app/src/pages/_app.tsx b/projects/app/src/pages/_app.tsx
@@ -93,7 +93,7 @@ function App({ Component, pageProps }: AppProps) {
   return (
     <>
       <Head>
-        <title>{feConfigs?.systemTitle || process.env.SYSTEM_NAME || 'GPT'}</title>
+        <title>{feConfigs?.systemTitle || process.env.SYSTEM_NAME || ''}</title>
         <meta
           name="description"
           content="FastGPT 是一个大模型应用编排系统，提供开箱即用的数据处理、模型调用等能力，可以快速的构建知识库并通过 Flow 可视化进行工作流编排，实现复杂的知识库场景！"