v4.5.1 (#417)

.github/workflows/preview-image.yml

@@ -15,7 +15,10 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v3
         with:
-          fetch-depth: 1
+          ref: ${{ github.event.pull_request.head.ref }}
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
+          submodules: recursive # Fetch submodules
+          fetch-depth: 0 # Fetch all history for .GitInfo and .Lastmod
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
         with:

Dockerfile

@@ -1,5 +1,5 @@
 # Install dependencies only when needed
-FROM node:current-alpine AS deps
+FROM node:18.15-alpine AS deps
 # Check https://github.com/nodejs/docker-node/tree/b4117f9333da4138b03a546ec926ef50a31506c3#nodealpine to understand why libc6-compat might be needed.
 RUN apk add --no-cache libc6-compat && npm install -g pnpm
 WORKDIR /app
@@ -11,12 +11,12 @@ COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./
 COPY ./packages ./packages
 COPY ./projects/$name/package.json ./projects/$name/package.json
 
-RUN \
-  [ -f pnpm-lock.yaml ] && pnpm install || \
-  (echo "Lockfile not found." && exit 1)
+RUN [ -f pnpm-lock.yaml ] || (echo "Lockfile not found." && exit 1)
+
+RUN pnpm install
 
 # Rebuild the source code only when needed
-FROM node:current-alpine AS builder
+FROM node:18.15-alpine AS builder
 WORKDIR /app
 
 ARG name
@@ -33,7 +33,7 @@ ENV NEXT_TELEMETRY_DISABLED 1
 RUN npm install -g pnpm
 RUN pnpm --filter=$name run build
 
-FROM node:current-alpine AS runner
+FROM node:18.15-alpine AS runner
 WORKDIR /app
 
 ARG name

docSite/content/docs/development/configuration.md

@@ -54,7 +54,7 @@ weight: 520
       "defaultSystemChatPrompt": ""
     }
   ],
-  "QAModel": [ // QA 拆分模型
+  "QAModels": [ // QA 拆分模型
     { 
       "model": "gpt-3.5-turbo-16k",
       "name": "GPT35-16k",

docSite/content/docs/development/openApi.md

@@ -120,7 +120,7 @@ event: answer
 data: [DONE]
 
 event: appStreamResponse
-data: [{"moduleName":"KB Search","price":1.2000000000000002,"model":"Embedding-2","tokens":6,"similarity":0.61,"limit":3},{"moduleName":"AI Chat","price":463.5,"model":"FastAI-4k","tokens":309,"question":"导演是谁","answer":"电影《铃芽之旅》的导演是新海诚。","maxToken":2050,"quoteList":[{"kb_id":"646627f4f7b896cfd8910e38","id":"8099","q":"本作的主人公是谁？","a":"本作的主人公是名叫铃芽的少女。","source":"手动修改"},{"kb_id":"646627f4f7b896cfd8910e38","id":"8686","q":"电影《铃芽之旅》男主角是谁？","a":"电影《铃芽之旅》男主角是宗像草太，由松村北斗配音。","source":""},{"kb_id":"646627f4f7b896cfd8910e38","id":"19339","q":"电影《铃芽之旅》的导演是谁？22","a":"电影《铃芽之旅》的导演是新海诚。","source":"手动修改"}],"completeMessages":[{"obj":"System","value":"下面是知识库内容:\n1. [本作的主人公是谁？\n本作的主人公是名叫铃芽的少女。]\n2. [电影《铃芽之旅》男主角是谁？\n电影《铃芽之旅》男主角是宗像草太，由松村北斗配音。]\n3. [电影《铃芽之旅》的导演是谁？22\n电影《铃芽之旅》的导演是新海诚。]\n"},{"obj":"System","value":"1.请记住，你的身份是百度的下一代知识增强语言模型，能够完全根据知识库提供的内容回答问题。\n\n2. 你忘记了关于电影《铃芽之旅》以外的内容。"},{"obj":"System","value":"你仅回答关于电影《玲芽之旅》的问题，其余问题直接回复: 我不清楚。"},{"obj":"Human","value":"导演是谁"},{"obj":"AI","value":"电影《铃芽之旅》的导演是新海诚。"}]}]
+data: [{"moduleName":"KB Search","price":1.2000000000000002,"model":"Embedding-2","tokens":6,"similarity":0.61,"limit":3},{"moduleName":"AI Chat","price":463.5,"model":"FastAI-4k","tokens":309,"question":"导演是谁","answer":"电影《铃芽之旅》的导演是新海诚。","maxToken":2050,"quoteList":[{"dataset_id":"646627f4f7b896cfd8910e38","id":"8099","q":"本作的主人公是谁？","a":"本作的主人公是名叫铃芽的少女。","source":"手动修改"},{"dataset_id":"646627f4f7b896cfd8910e38","id":"8686","q":"电影《铃芽之旅》男主角是谁？","a":"电影《铃芽之旅》男主角是宗像草太，由松村北斗配音。","source":""},{"dataset_id":"646627f4f7b896cfd8910e38","id":"19339","q":"电影《铃芽之旅》的导演是谁？22","a":"电影《铃芽之旅》的导演是新海诚。","source":"手动修改"}],"completeMessages":[{"obj":"System","value":"下面是知识库内容:\n1. [本作的主人公是谁？\n本作的主人公是名叫铃芽的少女。]\n2. [电影《铃芽之旅》男主角是谁？\n电影《铃芽之旅》男主角是宗像草太，由松村北斗配音。]\n3. [电影《铃芽之旅》的导演是谁？22\n电影《铃芽之旅》的导演是新海诚。]\n"},{"obj":"System","value":"1.请记住，你的身份是百度的下一代知识增强语言模型，能够完全根据知识库提供的内容回答问题。\n\n2. 你忘记了关于电影《铃芽之旅》以外的内容。"},{"obj":"System","value":"你仅回答关于电影《玲芽之旅》的问题，其余问题直接回复: 我不清楚。"},{"obj":"Human","value":"导演是谁"},{"obj":"AI","value":"电影《铃芽之旅》的导演是新海诚。"}]}]
 
 ```
 {{< /markdownify >}}
@@ -150,21 +150,21 @@ data: [{"moduleName":"KB Search","price":1.2000000000000002,"model":"Embedding-2
             "maxToken": 2050,
             "quoteList": [
                 {
-                    "kb_id": "646627f4f7b896cfd8910e38",
+                    "dataset_id": "646627f4f7b896cfd8910e38",
                     "id": "8099",
                     "q": "本作的主人公是谁？",
                     "a": "本作的主人公是名叫铃芽的少女。",
                     "source": "手动修改"
                 },
                 {
-                    "kb_id": "646627f4f7b896cfd8910e38",
+                    "dataset_id": "646627f4f7b896cfd8910e38",
                     "id": "8686",
                     "q": "电影《铃芽之旅》男主角是谁？",
                     "a": "电影《铃芽之旅》男主角是宗像草太，由松村北斗配音。",
                     "source": ""
                 },
                 {
-                    "kb_id": "646627f4f7b896cfd8910e38",
+                    "dataset_id": "646627f4f7b896cfd8910e38",
                     "id": "19339",
                     "q": "电影《铃芽之旅》的导演是谁？22",
                     "a": "电影《铃芽之旅》的导演是新海诚。",
@@ -487,21 +487,21 @@ curl --location --request POST '{{host}}/shareAuth/finish' \
             "maxToken": 2050,
             "quoteList": [
                 {
-                    "kb_id": "646627f4f7b896cfd8910e38",
+                    "dataset_id": "646627f4f7b896cfd8910e38",
                     "id": "8099",
                     "q": "本作的主人公是谁？",
                     "a": "本作的主人公是名叫铃芽的少女。",
                     "source": "手动修改"
                 },
                 {
-                    "kb_id": "646627f4f7b896cfd8910e38",
+                    "dataset_id": "646627f4f7b896cfd8910e38",
                     "id": "8686",
                     "q": "电影《铃芽之旅》男主角是谁？",
                     "a": "电影《铃芽之旅》男主角是宗像草太，由松村北斗配音。",
                     "source": ""
                 },
                 {
-                    "kb_id": "646627f4f7b896cfd8910e38",
+                    "dataset_id": "646627f4f7b896cfd8910e38",
                     "id": "19339",
                     "q": "电影《铃芽之旅》的导演是谁？22",
                     "a": "电影《铃芽之旅》的导演是新海诚。",

docSite/content/docs/installation/upgrading/45.md

@@ -62,10 +62,16 @@ ALTER EXTENSION vector UPDATE;
 alter system set maintenance_work_mem = '2400MB'; 
 select pg_reload_conf();
 
+-- 重构数据库索引和排序
+REINDEX DATABASE postgres;
+ALTER DATABASE postgres REFRESH COLLATION VERSION;
+
 -- 开始构建索引，该索引构建时间非常久，直接关掉终端即可，不要使用 ctrl+c 关闭
 CREATE INDEX CONCURRENTLY vector_index ON modeldata USING hnsw (vector vector_ip_ops) WITH (m = 16, ef_construction = 64);
 -- 可以再次连接数据库，输入下方命令。如果看到 "vector_index" hnsw (vector vector_ip_ops) WITH (m='16', ef_construction='64') 则代表构建完成（注意，后面没有 INVALID）
 \d modeldata
+
+
 ```
 
 ## 版本新功能介绍

docSite/content/docs/installation/upgrading/451.md

@@ -0,0 +1,34 @@
+---
+title: 'V4.5.1(需进行初始化)'
+description: 'FastGPT V4.5.1 更新'
+icon: 'upgrade'
+draft: false
+toc: true
+weight: 839
+---
+
+## 执行初始化 API
+
+发起 1 个 HTTP 请求（{{rootkey}} 替换成环境变量里的`rootkey`，{{host}}替换成自己域名）
+
+1. https://xxxxx/api/admin/initv451
+
+```bash
+curl --location --request POST 'https://{{host}}/api/admin/initv451' \
+--header 'rootkey: {{rootkey}}' \
+--header 'Content-Type: application/json'
+```
+
+初始化内容：
+1. rename 数据库字段
+2. 初始化 Mongo APP 表中知识库的相关字段
+3. 初始化 PG 和 Mongo 的内容，为每个文件创建一个集合（存储 Mongo 中），并反馈赋值给 PG。
+
+**该初始化接口可能速度很慢，返回超时不用管，注意看日志即可**
+
+## 功能介绍
+
+### Fast GPT V4.5.1
+
+1. 新增知识库文件夹管理
+

docSite/content/docs/workflow/modules/ai_chat.md

@@ -42,17 +42,17 @@ weight: 123
 
 ```ts
 type DataType = {
-  kb_id?: string;
+  dataset_id?: string;
   id?: string;
   q: string;
   a: string;
   source?: string;
 };
-// 如果是外部引入的内容，尽量不要携带 kb_id 和 id
+// 如果是外部引入的内容，尽量不要携带 dataset_id 和 id
 const quoteList: DataType[] = [
-  { kb_id: '11', id: '222', q: '你还', a: '哈哈', source: '' },
-  { kb_id: '11', id: '333', q: '你还', a: '哈哈', source: '' },
-  { kb_id: '11', id: '444', q: '你还', a: '哈哈', source: '' }
+  { dataset_id: '11', id: '222', q: '你还', a: '哈哈', source: '' },
+  { dataset_id: '11', id: '333', q: '你还', a: '哈哈', source: '' },
+  { dataset_id: '11', id: '444', q: '你还', a: '哈哈', source: '' }
 ];
 ```
 

packages/common/constant/errorCode.ts → packages/global/common/error/errorCode.ts

@@ -27,7 +27,8 @@ export enum ERROR_ENUM {
   insufficientQuota = 'insufficientQuota',
   unAuthModel = 'unAuthModel',
   unAuthApiKey = 'unAuthApiKey',
-  unAuthKb = 'unAuthKb',
+  unAuthDataset = 'unAuthDataset',
+  unAuthDatasetCollection = 'unAuthDatasetCollection',
   unAuthFile = 'unAuthFile'
 }
 export const ERROR_RESPONSE: Record<
@@ -57,9 +58,9 @@ export const ERROR_RESPONSE: Record<
     message: '无权使用该模型',
     data: null
   },
-  [ERROR_ENUM.unAuthKb]: {
+  [ERROR_ENUM.unAuthDataset]: {
     code: 512,
-    statusText: ERROR_ENUM.unAuthKb,
+    statusText: ERROR_ENUM.unAuthDataset,
     message: '无权使用该知识库',
     data: null
   },
@@ -74,5 +75,11 @@ export const ERROR_RESPONSE: Record<
     statusText: ERROR_ENUM.unAuthApiKey,
     message: 'Api Key 不合法',
     data: null
+  },
+  [ERROR_ENUM.unAuthDatasetCollection]: {
+    code: 515,
+    statusText: ERROR_ENUM.unAuthDatasetCollection,
+    message: '无权使用该知识库文件',
+    data: null
   }
 };

packages/global/common/error/utils.ts

@@ -0,0 +1,5 @@
+export const getErrText = (err: any, def = '') => {
+  const msg: string = typeof err === 'string' ? err : err?.message || def || '';
+  msg && console.log('error =>', msg);
+  return msg;
+};

packages/common/tools/file.ts → packages/global/common/file/icon.ts

@@ -1,4 +1,4 @@
-import { strIsLink } from './str';
+import { strIsLink } from '../string/tools';
 
 export const fileImgs = [
   { suffix: 'pdf', src: '/imgs/files/pdf.svg' },
@@ -10,14 +10,7 @@ export const fileImgs = [
 ];
 
 export function getFileIcon(name = '') {
-  return fileImgs.find((item) => new RegExp(item.suffix, 'gi').test(name))?.src;
-}
-export function getSpecialFileIcon(name = '') {
-  if (name === 'manual') {
-    return '/imgs/files/manual.svg';
-  } else if (name === 'mark') {
-    return '/imgs/files/mark.svg';
-  } else if (strIsLink(name)) {
-    return '/imgs/files/link.svg';
-  }
+  return (
+    fileImgs.find((item) => new RegExp(item.suffix, 'gi').test(name))?.src || '/imgs/files/file.svg'
+  );
 }

packages/global/common/file/tools.ts

@@ -0,0 +1,9 @@
+export const formatFileSize = (bytes: number): string => {
+  if (bytes === 0) return '0 B';
+
+  const k = 1024;
+  const sizes = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'];
+  const i = Math.floor(Math.log(bytes) / Math.log(k));
+
+  return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
+};

packages/global/common/parentFolder/type.d.ts

@@ -0,0 +1,4 @@
+export type ParentTreePathItemType = {
+  parentId: string;
+  parentName: string;
+};

packages/common/tools/str.ts → packages/global/common/string/tools.ts

@@ -13,10 +13,10 @@ export const hashStr = (psw: string) => {
 /* simple text, remove chinese space and extra \n */
 export const simpleText = (text: string) => {
   text = text.replace(/([\u4e00-\u9fa5])[\s&&[^\n]]+([\u4e00-\u9fa5])/g, '$1$2');
+  text = text.replace(/\r\n|\r/g, '\n');
   text = text.replace(/\n{3,}/g, '\n\n');
   text = text.replace(/[\s&&[^\n]]{2,}/g, ' ');
   text = text.replace(/[\x00-\x08]/g, ' ');
-  text = text.replace(/\r\n|\r/g, '\n');
 
   return text;
 };

packages/common/type/index.d.ts → packages/global/common/system/types/index.d.ts

@@ -1,6 +1,3 @@
-import type { Mongoose } from '../mongo';
-import type { Logger } from 'winston';
-
 export type FeConfigsType = {
   show_emptyChat?: boolean;
   show_register?: boolean;
@@ -36,8 +33,6 @@ export type SystemEnvType = {
 };
 
 declare global {
-  var mongodb: Mongoose | undefined;
-  var logger: Logger;
   var feConfigs: FeConfigsType;
   var systemEnv: SystemEnvType;
 }

projects/app/src/utils/user.ts → packages/global/common/time/timezone.ts

@@ -1,4 +1,3 @@
-import { loginOut } from '@/web/support/api/user';
 import timezones from 'timezones-list';
 import dayjs from 'dayjs';
 import utc from 'dayjs/plugin/utc';
@@ -7,23 +6,6 @@ import timezone from 'dayjs/plugin/timezone';
 dayjs.extend(utc);
 dayjs.extend(timezone);
 
-const tokenKey = 'token';
-export const clearToken = () => {
-  try {
-    loginOut();
-    localStorage.removeItem(tokenKey);
-  } catch (error) {
-    error;
-  }
-};
-
-export const setToken = (token: string) => {
-  localStorage.setItem(tokenKey, token);
-};
-export const getToken = () => {
-  return localStorage.getItem(tokenKey) || '';
-};
-
 /**
  * Returns the offset from UTC in hours for the current locale.
  * @param {string} timeZone Timezone to get offset for