feat: url fetch and create file (#199)

* docs

* docs

* feat: url fetch and create file

README.md

@@ -119,3 +119,4 @@ FastGPT 是一个基于 LLM 大语言模型的知识库问答系统，提供开
 1. 允许作为后台服务直接商用，但不允许直接使用 saas 服务商用。
 2. 需保留相关版权信息。
 3. 完整请查看 [FstGPT Open Source License](./LICENSE)
+4. 联系方式：[email protected], [点击查看定价策略](https://fael3z0zfze.feishu.cn/docx/F155dbirfo8vDDx2WgWc6extnwf)

client/public/locales/en/common.json

@@ -54,9 +54,14 @@
   },
   "file": {
     "Click to download CSV template": "Click to download CSV template",
-    "Drag and drop": "Drag and drop files here, or click",
+    "Create File": "Create File",
+    "Create file": "Create file",
+    "Drag and drop": "Drag and drop files here",
+    "Fetch Url": "Fetch Url",
     "If the imported file is garbled, please convert CSV to UTF-8 encoding format": "If the imported file is garbled, please convert CSV to UTF-8 encoding format",
     "Release the mouse to upload the file": "Release the mouse to upload the file",
+    "Select a maximum of 10 files": "Select a maximum of 10 files",
+    "max 10": "Max 10 files",
     "select a document": "select a document",
     "support": "support {{fileExtension}} file",
     "upload error description": "Only upload multiple files or one folder at a time"

client/public/locales/zh/common.json

@@ -54,9 +54,14 @@
   },
   "file": {
     "Click to download CSV template": "点击下载 CSV 模板",
-    "Drag and drop": "拖拽文件至此，或点击",
+    "Create File": "创建新文件",
+    "Create file": "创建文件",
+    "Drag and drop": "拖拽文件至此",
+    "Fetch Url": "链接读取",
     "If the imported file is garbled, please convert CSV to UTF-8 encoding format": "如果导入文件乱码，请将 CSV 转成 UTF-8 编码格式",
     "Release the mouse to upload the file": "松开鼠标上传文件",
+    "Select a maximum of 10 files": "最多选择10个文件",
+    "max 10": "最多选择 10 个文件",
     "select a document": "选择文件",
     "support": "支持 {{fileExtension}} 文件",
     "upload error description": "单次只支持上传多个文件或者一个文件夹"

client/src/api/plugins/common.ts

@@ -0,0 +1,6 @@
+import { GET, POST, PUT, DELETE } from '../request';
+
+import type { FetchResultItem } from '@/types/plugin';
+
+export const fetchUrls = (urlList: string[]) =>
+  POST<FetchResultItem[]>(`/plugins/urlFetch`, { urlList });

client/src/constants/flow/ModuleTemplate.ts

@@ -182,7 +182,7 @@ export const ChatModule: FlowModuleTemplateType = {
     {
       key: TaskResponseKeyEnum.answerText,
       label: '模型回复',
-      description: '如果外接了内容，会在回复结束时自动添加\n\n',
+      description: '将在 stream 回复完毕后触发',
       valueType: FlowValueTypeEnum.string,
       type: FlowOutputItemTypeEnum.source,
       targets: []

client/src/hooks/useSelectFile.tsx

@@ -1,8 +1,12 @@
 import React, { useRef, useCallback } from 'react';
 import { Box } from '@chakra-ui/react';
+import { useToast } from './useToast';
+import { useTranslation } from 'react-i18next';
 
 export const useSelectFile = (props?: { fileType?: string; multiple?: boolean }) => {
+  const { t } = useTranslation();
   const { fileType = '*', multiple = false } = props || {};
+  const { toast } = useToast();
   const SelectFileDom = useRef<HTMLInputElement>(null);
 
   const File = useCallback(
@@ -15,12 +19,18 @@ export const useSelectFile = (props?: { fileType?: string; multiple?: boolean })
           multiple={multiple}
           onChange={(e) => {
             if (!e.target.files || e.target.files?.length === 0) return;
+            if (e.target.files.length > 10) {
+              return toast({
+                status: 'warning',
+                title: t('file.Select a maximum of 10 files')
+              });
+            }
             onSelect(Array.from(e.target.files));
           }}
         />
       </Box>
     ),
-    [fileType, multiple]
+    [fileType, multiple, t, toast]
   );
 
   const onOpen = useCallback(() => {

client/src/pages/api/plugins/urlFetch.ts

@@ -5,12 +5,9 @@ import { JSDOM } from 'jsdom';
 import { Readability } from '@mozilla/readability';
 import { jsonRes } from '@/service/response';
 import { authUser } from '@/service/utils/auth';
+import type { FetchResultItem } from '@/types/plugin';
+import { simpleText } from '@/utils/file';
 
-type FetchResultItem = {
-  url: string;
-  title: string;
-  content: string;
-};
 export type UrlFetchResponse = FetchResultItem[];
 
 const fetchContent = async (req: NextApiRequest, res: NextApiResponse) => {
@@ -38,10 +35,11 @@ const fetchContent = async (req: NextApiRequest, res: NextApiResponse) => {
           const reader = new Readability(dom.window.document);
           const article = reader.parse();
 
+          const content = article?.textContent || '';
+
           return {
             url,
-            title: article?.title || '',
-            content: article?.textContent || ''
+            content: simpleText(`${article?.title}\n${content}`)
           };
         })
       )

client/src/pages/kb/detail/components/DataCard.tsx

@@ -217,7 +217,7 @@ const DataCard = ({ kbId }: { kbId: string }) => {
               <Box color={'myGray.600'}>{item.a}</Box>
             </Box>
             <Flex py={2} px={4} h={'36px'} alignItems={'flex-end'} fontSize={'sm'}>
-              <Box className={'textEllipsis'} flex={1}>
+              <Box className={'textEllipsis'} flex={1} color={'myGray.500'}>
                 {item.source?.trim()}
               </Box>
               <IconButton