4.8.5 test (#1805)

* perf: revert tip

* feat: create copy app

* perf: file stream read

* perf: read directory over 100 files

* perf: index

* fix: team chat api error

* lock

* fix: i18n file

.vscode/settings.json

@@ -4,7 +4,7 @@
     "typescript.tsdk": "node_modules/typescript/lib",
     "prettier.prettierPath": "",
     "i18n-ally.localesPaths": [
-        "projects/app/i18n",
+        "packages/web/i18n",
     ],
     "i18n-ally.enabledParsers": ["json", "yaml", "js", "ts"],
     "i18n-ally.keystyle": "nested",

docSite/content/zh-cn/docs/development/upgrading/485.md

@@ -32,6 +32,10 @@ curl --location --request POST 'https://{{host}}/api/admin/initv485' \
 ## V4.8.5 更新说明
 
 1. 新增 - 合并插件和应用，统一成工作台
-2. 修复 - SSR渲染
-3. 修复 - 定时任务无法实际关闭
-4. 修复 - 输入引导特殊字符导致正则报错
+2. 新增 - 应用创建副本功能
+3. 优化 - 原文件编码存取
+4. 优化 - 文件夹读取，支持单个文件夹超出 100 个文件
+5. 优化 - 问答拆分/手动录入，当有`a`字段时，自动将`q`作为补充索引。
+6. 修复 - SSR渲染
+7. 修复 - 定时任务无法实际关闭
+8. 修复 - 输入引导特殊字符导致正则报错

package.json

@@ -21,12 +21,6 @@
     "react-i18next": "13.5.0",
     "zhlint": "^0.7.1"
   },
-  "resolutions": {
-    "react": "18.3.1",
-    "react-dom": "18.3.1",
-    "@types/react": "18.3.0",
-    "@types/react-dom": "18.3.0"
-  },
   "lint-staged": {
     "./**/**/*.{ts,tsx,scss}": "npm run format-code",
     "./docSite/**/**/*.md": "npm run format-doc"

packages/global/common/string/textSplitter.ts

@@ -350,7 +350,7 @@ export const splitText2Chunks = (props: SplitProps): SplitResponse => {
 
     return commonSplit(props);
   });
-  console.log(Date.now() - start);
+
   return {
     chunks: splitResult.map((item) => item.chunks).flat(),
     chars: splitResult.reduce((sum, item) => sum + item.chars, 0)

packages/global/core/dataset/type.d.ts

@@ -179,10 +179,9 @@ export type DatasetFileSchema = {
   filename: string;
   contentType: string;
   metadata: {
-    contentType: string;
-    datasetId: string;
     teamId: string;
     tmbId: string;
+    encoding?: string;
   };
 };
 

packages/global/package.json

@@ -17,6 +17,6 @@
   },
   "devDependencies": {
     "@types/js-yaml": "^4.0.9",
-    "@types/node": "^20.8.5"
+    "@types/node": "^20.14.2"
   }
 }

packages/plugins/package.json

@@ -3,7 +3,7 @@
   "version": "1.0.0",
   "dependencies": {},
   "devDependencies": {
-    "@types/node": "^20.8.5",
+    "@types/node": "^20.14.2",
     "@fastgpt/global": "workspace:*",
     "@fastgpt/service": "workspace:*"
   }

packages/service/common/file/gridfs/controller.ts

@@ -8,7 +8,8 @@ import { detectFileEncoding } from '@fastgpt/global/common/file/tools';
 import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
 import { MongoRawTextBuffer } from '../../buffer/rawText/schema';
 import { readRawContentByFileBuffer } from '../read/utils';
-import { gridFsStream2Buffer } from './utils';
+import { gridFsStream2Buffer, stream2Encoding } from './utils';
+import { addLog } from '../../system/log';
 
 export function getGFSCollection(bucket: `${BucketNameEnum}`) {
   MongoFileSchema;
@@ -44,8 +45,11 @@ export async function uploadFile({
   const stats = await fsp.stat(path);
   if (!stats.isFile()) return Promise.reject(`${path} is not a file`);
 
+  const { stream: readStream, encoding } = await stream2Encoding(fs.createReadStream(path));
+
   metadata.teamId = teamId;
   metadata.tmbId = tmbId;
+  metadata.encoding = encoding;
 
   // create a gridfs bucket
   const bucket = getGridBucket(bucketName);
@@ -57,7 +61,7 @@ export async function uploadFile({
 
   // save to gridfs
   await new Promise((resolve, reject) => {
-    fs.createReadStream(path)
+    readStream
       .pipe(stream as any)
       .on('finish', resolve)
       .on('error', reject);
@@ -113,19 +117,8 @@ export async function getDownloadStream({
   fileId: string;
 }) {
   const bucket = getGridBucket(bucketName);
-  const encodeStream = bucket.openDownloadStream(new Types.ObjectId(fileId), { end: 100 });
-  const rawStream = bucket.openDownloadStream(new Types.ObjectId(fileId));
-
-  /* get encoding */
-  const buffer = await gridFsStream2Buffer(encodeStream);
 
-  const encoding = detectFileEncoding(buffer);
-
-  return {
-    fileStream: rawStream,
-    encoding
-    // encoding: 'utf-8'
-  };
+  return bucket.openDownloadStream(new Types.ObjectId(fileId));
 }
 
 export const readFileContentFromMongo = async ({
@@ -150,9 +143,8 @@ export const readFileContentFromMongo = async ({
       filename: fileBuffer.metadata?.filename || ''
     };
   }
-  const start = Date.now();
 
-  const [file, { encoding, fileStream }] = await Promise.all([
+  const [file, fileStream] = await Promise.all([
     getFileById({ bucketName, fileId }),
     getDownloadStream({ bucketName, fileId })
   ]);
@@ -163,8 +155,11 @@ export const readFileContentFromMongo = async ({
 
   const extension = file?.filename?.split('.')?.pop()?.toLowerCase() || '';
 
+  const start = Date.now();
   const fileBuffers = await gridFsStream2Buffer(fileStream);
-  // console.log('get file buffer', Date.now() - start);
+  addLog.debug('get file buffer', { time: Date.now() - start });
+
+  const encoding = file?.metadata?.encoding || detectFileEncoding(fileBuffers);
 
   const { rawText } = await readRawContentByFileBuffer({
     extension,
@@ -177,7 +172,8 @@ export const readFileContentFromMongo = async ({
     }
   });
 
-  if (rawText.trim()) {
+  // < 14M
+  if (fileBuffers.length < 14 * 1024 * 1024 && rawText.trim()) {
     MongoRawTextBuffer.create({
       sourceId: fileId,
       rawText,

packages/service/common/file/gridfs/utils.ts

@@ -1,3 +1,6 @@
+import { detectFileEncoding } from '@fastgpt/global/common/file/tools';
+import { PassThrough } from 'stream';
+
 export const gridFsStream2Buffer = (stream: NodeJS.ReadableStream) => {
   return new Promise<Buffer>((resolve, reject) => {
     let tmpBuffer: Buffer = Buffer.from([]);
@@ -13,3 +16,38 @@ export const gridFsStream2Buffer = (stream: NodeJS.ReadableStream) => {
     });
   });
 };
+
+export const stream2Encoding = async (stream: NodeJS.ReadableStream) => {
+  const start = Date.now();
+  const copyStream = stream.pipe(new PassThrough());
+
+  /* get encoding */
+  const buffer = await (() => {
+    return new Promise<Buffer>((resolve, reject) => {
+      let tmpBuffer: Buffer = Buffer.from([]);
+
+      stream.on('data', (chunk) => {
+        if (tmpBuffer.length < 200) {
+          tmpBuffer = Buffer.concat([tmpBuffer, chunk]);
+
+          if (tmpBuffer.length >= 200) {
+            resolve(tmpBuffer);
+          }
+        }
+      });
+      stream.on('end', () => {
+        resolve(tmpBuffer);
+      });
+      stream.on('error', (err) => {
+        reject(err);
+      });
+    });
+  })();
+
+  const enc = detectFileEncoding(buffer);
+  console.log('Get encoding time', Date.now() - start, enc);
+  return {
+    encoding: enc,
+    stream: copyStream
+  };
+};

packages/service/package.json

@@ -15,12 +15,12 @@
     "decompress": "^4.2.1",
     "domino-ext": "^2.1.4",
     "encoding": "^0.1.13",
-    "lodash": "^4.17.21",
     "file-type": "^19.0.0",
     "iconv-lite": "^0.6.3",
     "joplin-turndown-plugin-gfm": "^1.0.12",
     "json5": "^2.2.3",
     "jsonwebtoken": "^9.0.2",
+    "lodash": "^4.17.21",
     "mammoth": "^1.6.0",
     "mongoose": "^7.0.2",
     "multer": "1.4.5-lts.1",
@@ -39,10 +39,10 @@
     "@types/cookie": "^0.5.2",
     "@types/decompress": "^4.2.7",
     "@types/jsonwebtoken": "^9.0.3",
+    "@types/lodash": "^4.14.191",
     "@types/multer": "^1.4.10",
     "@types/node-cron": "^3.0.11",
     "@types/papaparse": "5.3.7",
-    "@types/lodash": "^4.14.191",
     "@types/pg": "^8.6.6",
     "@types/tunnel": "^0.0.4",
     "@types/turndown": "^5.0.4"

packages/web/components/common/MyMenu/index.tsx

@@ -46,12 +46,26 @@ const MyMenu = ({
       _hover: {
         backgroundColor: 'primary.50',
         color: 'primary.600'
+      },
+      _focus: {
+        backgroundColor: 'primary.50',
+        color: 'primary.600'
+      },
+      _active: {
+        backgroundColor: 'primary.50',
+        color: 'primary.600'
       }
     },
     danger: {
       color: 'red.600',
       _hover: {
         background: 'red.1'
+      },
+      _focus: {
+        background: 'red.1'
+      },
+      _active: {
+        background: 'red.1'
       }
     }
   };

packages/web/components/common/MyPopover/PopoverConfirm.tsx

@@ -76,7 +76,7 @@ const PopoverConfirm = ({
       <PopoverContent p={4}>
         <PopoverArrow />
 
-        <HStack alignItems={'flex-start'}>
+        <HStack alignItems={'flex-start'} color={'myGray.700'}>
           <MyIcon name={map.icon as any} w={'1.5rem'} />
           <Box fontSize={'sm'}>{content}</Box>
         </HStack>

projects/app/i18n/en/app.json → packages/web/i18n/en/app.json

@@ -9,11 +9,14 @@
   "Chat Logs Tips": "Logs will record online, shared and API (chatId required) conversation records for this app",
   "Chat logs": "Chat Logs",
   "Confirm Del App Tip": "Confirm to delete this app and all its chat records?",
+  "Confirm copy app tip": "The system will create an application with the same configuration for you, but the permission will not be copied, please confirm!",
   "Confirm delete folder tip": "Are you sure to delete this folder? All the following applications and corresponding chat records will be deleted, please confirm!",
   "Connection is invalid": "Connection is invalid",
   "Connection type is different": "Connection type is different",
   "Copy Module Config": "Copy Config",
+  "Copy one app": "Copy",
   "Create bot": "App",
+  "Create copy success": "Create copy success",
   "Create one ai app": "Create AI app",
   "Current settings": "Current settings",
   "Dataset Quote Template": "Knowledge Base QA Mode",
@@ -77,5 +80,8 @@
     "Plugin": "Plugin",
     "Simple bot": "Simple bot",
     "Workflow bot": "Workflow"
+  },
+  "version": {
+    "Revert success": "Revert success"
   }
 }

projects/app/i18n/zh/app.json → packages/web/i18n/zh/app.json

@@ -8,11 +8,14 @@
   "Chat Logs Tips": "日志会记录该应用的在线、分享和 API(需填写 chatId) 对话记录",
   "Chat logs": "对话日志",
   "Confirm Del App Tip": "确认删除该应用及其所有聊天记录？",
+  "Confirm copy app tip": "系统将为您创建一个相同配置应用，但权限不会进行复制，请确认！",
   "Confirm delete folder tip": "确认删除该文件夹？将会删除它下面所有应用及对应的聊天记录，请确认！",
   "Connection is invalid": "连接无效",
   "Connection type is different": "连接的类型不一致",
   "Copy Module Config": "复制配置",
+  "Copy one app": "创建副本",
   "Create bot": "应用",
+  "Create copy success": "创建副本成功",
   "Create one ai app": "创建一个AI应用",
   "Current settings": "当前配置",
   "Dataset Quote Template": "知识库问答模式",
@@ -76,5 +79,8 @@
     "Plugin": "插件",
     "Simple bot": "简易应用",
     "Workflow bot": "工作流"
+  },
+  "version": {
+    "Revert success": "回滚成功"
   }
 }

projects/app/i18n/zh/common.json → packages/web/i18n/zh/common.json

@@ -252,6 +252,7 @@
       "Publish Confirm": "确认发布应用？会立即更新所有发布渠道的应用状态。",
       "Publish Failed": "发布失败",
       "Publish Success": "发布成功",
+      "Publish app tip": "发布应用后，所有发布渠道将会立即使用该版本",
       "Question Guide": "猜你想问",
       "Question Guide Tip": "对话结束后，会为生成 3 个引导性问题。",
       "Quote prompt": "引用模板提示词",