fix: reg failed

labring · Aug 31, 2023 · 2ae8d43 · 2ae8d43
1 parent 0ea464f
commit 2ae8d43
Show file tree

Hide file tree

Showing 3 changed files with 74 additions and 69 deletions.
diff --git a/client/src/constants/common.ts b/client/src/constants/common.ts
@@ -6,11 +6,11 @@ export enum UserAuthTypeEnum {
 export const PRICE_SCALE = 100000;
 
 export const fileImgs = [
-  { reg: /pdf/gi, src: '/imgs/files/pdf.svg' },
-  { reg: /csv/gi, src: '/imgs/files/csv.svg' },
-  { reg: /(doc|docs)/gi, src: '/imgs/files/doc.svg' },
-  { reg: /txt/gi, src: '/imgs/files/txt.svg' },
-  { reg: /md/gi, src: '/imgs/files/markdown.svg' }
+  { suffix: 'pdf', src: '/imgs/files/pdf.svg' },
+  { suffix: 'csv', src: '/imgs/files/csv.svg' },
+  { suffix: '(doc|docs)', src: '/imgs/files/doc.svg' },
+  { suffix: 'txt', src: '/imgs/files/txt.svg' },
+  { suffix: 'md', src: '/imgs/files/markdown.svg' }
 ];
 
 export const htmlTemplate = `<!DOCTYPE html>

diff --git a/client/src/pages/kb/detail/components/Import/FileSelect.tsx b/client/src/pages/kb/detail/components/Import/FileSelect.tsx
@@ -76,76 +76,76 @@ const FileSelect = ({
       setSelecting(true);
       try {
         // Parse file by file
-        let promise = Promise.resolve<FileItemType[]>([]);
-        files.forEach((file) => {
-          promise = promise.then(async (result) => {
-            const extension = file?.name?.split('.')?.pop()?.toLowerCase();
+        const chunkFiles: FileItemType[] = [];
 
-            /* text file */
-            const icon = fileImgs.find((item) => new RegExp(item.reg).test(file.name))?.src;
-            let text = await (async () => {
-              switch (extension) {
-                case 'txt':
-                case 'md':
-                  return readTxtContent(file);
-                case 'pdf':
-                  return readPdfContent(file);
-                case 'doc':
-                case 'docx':
-                  return readDocContent(file);
-              }
-              return '';
-            })();
+        for await (let file of files) {
+          const extension = file?.name?.split('.')?.pop()?.toLowerCase();
 
-            if (!icon) return result;
+          /* text file */
+          const icon = fileImgs.find((item) => new RegExp(item.suffix, 'gi').test(file.name))?.src;
 
-            if (text) {
-              text = simpleText(text);
-              const splitRes = splitText2Chunks({
-                text,
-                maxLen: chunkLen
-              });
-              const fileItem: FileItemType = {
-                id: nanoid(),
-                filename: file.name,
-                icon,
-                text,
-                tokens: splitRes.tokens,
-                chunks: splitRes.chunks.map((chunk) => ({
-                  q: chunk,
-                  a: '',
-                  source: file.name
-                }))
-              };
-              return [fileItem].concat(result);
-            }
+          if (!icon) {
+            continue;
+          }
 
-            /* csv file */
-            if (extension === 'csv') {
-              const { header, data } = await readCsvContent(file);
-              if (header[0] !== 'question' || header[1] !== 'answer') {
-                throw new Error('csv 文件格式有误,请确保 question 和 answer 两列');
-              }
-              const fileItem: FileItemType = {
-                id: nanoid(),
-                filename: file.name,
-                icon,
-                tokens: 0,
-                text: '',
-                chunks: data.map((item) => ({
-                  q: item[0],
-                  a: item[1],
-                  source: item[2] || file.name
-                }))
-              };
-              return [fileItem].concat(result);
+          let text = await (async () => {
+            switch (extension) {
+              case 'txt':
+              case 'md':
+                return readTxtContent(file);
+              case 'pdf':
+                return readPdfContent(file);
+              case 'doc':
+              case 'docx':
+                return readDocContent(file);
             }
-            return result;
-          });
-        });
+            return '';
+          })();
 
-        const chunkFiles = await promise;
+          if (text) {
+            text = simpleText(text);
+            const splitRes = splitText2Chunks({
+              text,
+              maxLen: chunkLen
+            });
+            const fileItem: FileItemType = {
+              id: nanoid(),
+              filename: file.name,
+              icon,
+              text,
+              tokens: splitRes.tokens,
+              chunks: splitRes.chunks.map((chunk) => ({
+                q: chunk,
+                a: '',
+                source: file.name
+              }))
+            };
+            chunkFiles.unshift(fileItem);
+            continue;
+          }
+
+          /* csv file */
+          if (extension === 'csv') {
+            const { header, data } = await readCsvContent(file);
+            if (header[0] !== 'question' || header[1] !== 'answer') {
+              throw new Error('csv 文件格式有误,请确保 question 和 answer 两列');
+            }
+            const fileItem: FileItemType = {
+              id: nanoid(),
+              filename: file.name,
+              icon,
+              tokens: 0,
+              text: '',
+              chunks: data.map((item) => ({
+                q: item[0],
+                a: item[1],
+                source: item[2] || file.name
+              }))
+            };
 
+            chunkFiles.unshift(fileItem);
+          }
+        }
         onPushFiles(chunkFiles);
       } catch (error: any) {
         console.log(error);

diff --git a/client/src/service/events/generateQA.ts b/client/src/service/events/generateQA.ts
@@ -9,6 +9,7 @@ import { axiosConfig, getAIChatApi } from '../ai/openai';
 import { ChatCompletionRequestMessage } from 'openai';
 import { modelToolMap } from '@/utils/plugin';
 import { gptMessage2ChatType } from '@/utils/adapt';
+import { addLog } from '../utils/tools';
 
 const reduceQueue = () => {
   global.qaQueueLen = global.qaQueueLen > 0 ? global.qaQueueLen - 1 : 0;
@@ -105,12 +106,16 @@ A2:
             const result = formatSplitText(answer || ''); // 格式化后的QA对
             console.log(`split result length: `, result.length);
             // 计费
-            result.length > 0 &&
+            if (result.length > 0) {
               pushQABill({
                 userId: data.userId,
                 totalTokens,
                 appName: 'QA 拆分'
               });
+            } else {
+              addLog.info(`QA result 0:`, { answer });
+            }
+
             return {
               rawContent: answer,
               result