Snip test (#3204)

* fix: index * fix: snapshot error; perf: snapshot diff compare * perf: init simple edit history
labring · Nov 21, 2024 · 019bf67 · 019bf67
1 parent 9b2c3b2
commit 019bf67
Show file tree

Hide file tree

Showing 11 changed files with 218 additions and 168 deletions.
diff --git a/docSite/content/zh-cn/docs/development/upgrading/4814.md b/docSite/content/zh-cn/docs/development/upgrading/4814.md
@@ -14,4 +14,6 @@ weight: 810
 3. 新增 - 重写 chatContext，对话测试也会有日志，并且刷新后不会丢失对话。
 4. 新增 - 分享链接支持配置是否允许查看原文。
 5. 优化 - 工作流 ui 细节。
-6. 修复 - 分块策略，四级标题会被丢失。 同时新增了五级标题的支持。
+6. 优化 - 应用编辑记录采用 diff 存储，避免浏览器溢出。
+7. 修复 - 分块策略，四级标题会被丢失。 同时新增了五级标题的支持。
+8. 修复 - MongoDB 知识库集合唯一索引。
diff --git a/packages/service/core/dataset/collection/schema.ts b/packages/service/core/dataset/collection/schema.ts
@@ -118,7 +118,7 @@ try {
     {
       unique: true,
       partialFilterExpression: {
-        externalFileId: { $exists: true }
+        externalFileId: { $exists: true, $ne: '' }
       }
     }
   );

diff --git a/projects/app/src/pages/app/detail/components/Plugin/Header.tsx b/projects/app/src/pages/app/detail/components/Plugin/Header.tsx
@@ -13,11 +13,7 @@ import { useTranslation } from 'next-i18next';
 
 import MyIcon from '@fastgpt/web/components/common/Icon';
 import { useContextSelector } from 'use-context-selector';
-import {
-  WorkflowContext,
-  WorkflowSnapshotsType,
-  WorkflowStateType
-} from '../WorkflowComponents/context';
+import { WorkflowContext, WorkflowSnapshotsType } from '../WorkflowComponents/context';
 import { AppContext, TabEnum } from '../context';
 import RouteTab from '../RouteTab';
 import { useRouter } from 'next/router';
@@ -38,7 +34,7 @@ import {
   WorkflowInitContext
 } from '../WorkflowComponents/context/workflowInitContext';
 import { WorkflowEventContext } from '../WorkflowComponents/context/workflowEventContext';
-import { applyDiff } from '@/web/core/app/diff';
+import { getAppConfigByDiff } from '@/web/core/app/diff';
 
 const Header = () => {
   const { t } = useTranslation();
@@ -56,16 +52,19 @@ const Header = () => {
 
   const nodes = useContextSelector(WorkflowInitContext, (v) => v.nodes);
   const edges = useContextSelector(WorkflowNodeEdgeContext, (v) => v.edges);
-  const {
-    flowData2StoreData,
-    flowData2StoreDataAndCheck,
-    setWorkflowTestData,
-    past,
-    future,
-    setPast,
-    onSwitchTmpVersion,
-    onSwitchCloudVersion
-  } = useContextSelector(WorkflowContext, (v) => v);
+
+  const flowData2StoreData = useContextSelector(WorkflowContext, (v) => v.flowData2StoreData);
+  const flowData2StoreDataAndCheck = useContextSelector(
+    WorkflowContext,
+    (v) => v.flowData2StoreDataAndCheck
+  );
+  const setWorkflowTestData = useContextSelector(WorkflowContext, (v) => v.setWorkflowTestData);
+  const past = useContextSelector(WorkflowContext, (v) => v.past);
+  const future = useContextSelector(WorkflowContext, (v) => v.future);
+  const setPast = useContextSelector(WorkflowContext, (v) => v.setPast);
+  const onSwitchTmpVersion = useContextSelector(WorkflowContext, (v) => v.onSwitchTmpVersion);
+  const onSwitchCloudVersion = useContextSelector(WorkflowContext, (v) => v.onSwitchCloudVersion);
+
   const showHistoryModal = useContextSelector(WorkflowEventContext, (v) => v.showHistoryModal);
   const setShowHistoryModal = useContextSelector(
     WorkflowEventContext,
@@ -82,17 +81,19 @@ const Header = () => {
         past.find((snapshot) => snapshot.isSaved);
 
       const initialState = past[past.length - 1]?.state;
-      const savedSnapshotState = applyDiff(initialState, savedSnapshot?.diff);
+      const savedSnapshotState = getAppConfigByDiff(initialState, savedSnapshot?.diff);
 
       const val = compareSnapshot(
+        // nodes of the saved snapshot
         {
           nodes: savedSnapshotState?.nodes,
           edges: savedSnapshotState?.edges,
           chatConfig: savedSnapshotState?.chatConfig
         },
+        // nodes of the current canvas
         {
-          nodes: nodes,
-          edges: edges,
+          nodes,
+          edges,
           chatConfig: appDetail.chatConfig
         }
       );
@@ -140,8 +141,6 @@ const Header = () => {
 
   const onBack = useCallback(async () => {
     try {
-      localStorage.removeItem(`${appDetail._id}-past`);
-      localStorage.removeItem(`${appDetail._id}-future`);
       router.push({
         pathname: '/app/list',
         query: {
@@ -150,7 +149,7 @@ const Header = () => {
         }
       });
     } catch (error) {}
-  }, [appDetail._id, appDetail.parentId, lastAppListRouteType, router]);
+  }, [appDetail.parentId, lastAppListRouteType, router]);
 
   const Render = useMemo(() => {
     return (

diff --git a/projects/app/src/pages/app/detail/components/SimpleApp/Edit.tsx b/projects/app/src/pages/app/detail/components/SimpleApp/Edit.tsx
@@ -17,17 +17,44 @@ import styles from './styles.module.scss';
 import { useSystem } from '@fastgpt/web/hooks/useSystem';
 import { useTranslation } from 'next-i18next';
 import { onSaveSnapshotFnType, SimpleAppSnapshotType } from './useSnapshots';
-import { applyDiff } from '@/web/core/app/diff';
+import { getAppConfigByDiff, getAppDiffConfig } from '@/web/core/app/diff';
+import { formatTime2YMDHMS } from '@fastgpt/global/common/string/time';
+
+const convertOldFormatHistory = (past: SimpleAppSnapshotType[]) => {
+  const baseState = past[past.length - 1].appForm;
+
+  return past.map((item, index) => {
+    if (index === past.length - 1) {
+      return {
+        title: item.title,
+        isSaved: item.isSaved,
+        state: baseState
+      };
+    }
+
+    const currentState = item.appForm;
+
+    const diff = getAppDiffConfig(baseState, currentState);
+
+    return {
+      title: item.title || formatTime2YMDHMS(new Date()),
+      isSaved: item.isSaved,
+      diff
+    };
+  });
+};
 
 const Edit = ({
   appForm,
   setAppForm,
   past,
+  setPast,
   saveSnapshot
 }: {
   appForm: AppSimpleEditFormType;
   setAppForm: React.Dispatch<React.SetStateAction<AppSimpleEditFormType>>;
   past: SimpleAppSnapshotType[];
+  setPast: (value: React.SetStateAction<SimpleAppSnapshotType[]>) => void;
   saveSnapshot: onSaveSnapshotFnType;
 }) => {
   const { isPc } = useSystem();
@@ -40,19 +67,33 @@ const Edit = ({
     // show selected dataset
     loadAllDatasets();
 
-    const appForm = appWorkflow2Form({
-      nodes: appDetail.modules,
-      chatConfig: appDetail.chatConfig
-    });
+    if (appDetail.version !== 'v2') {
+      return setAppForm(
+        appWorkflow2Form({
+          nodes: v1Workflow2V2((appDetail.modules || []) as any)?.nodes,
+          chatConfig: appDetail.chatConfig
+        })
+      );
+    }
 
     // Get the latest snapshot
     if (past?.[0]?.diff) {
-      const pastState = applyDiff(past[past.length - 1].state, past[0].diff);
+      const pastState = getAppConfigByDiff(past[past.length - 1].state, past[0].diff);
 
       return setAppForm(pastState);
+    } else if (past && past.length > 0 && past?.every((item) => item.appForm)) {
+      // 格式化成 diff
+      const newPast = convertOldFormatHistory(past);
+
+      setPast(newPast);
+      return setAppForm(getAppConfigByDiff(newPast[newPast.length - 1].state, newPast[0].diff));
     }
 
-    setAppForm(appForm);
+    const appForm = appWorkflow2Form({
+      nodes: appDetail.modules,
+      chatConfig: appDetail.chatConfig
+    });
+
     // Set the first snapshot
     if (past.length === 0) {
       saveSnapshot({
@@ -62,14 +103,7 @@ const Edit = ({
       });
     }
 
-    if (appDetail.version !== 'v2') {
-      setAppForm(
-        appWorkflow2Form({
-          nodes: v1Workflow2V2((appDetail.modules || []) as any)?.nodes,
-          chatConfig: appDetail.chatConfig
-        })
-      );
-    }
+    setAppForm(appForm);
   });
 
   return (

diff --git a/projects/app/src/pages/app/detail/components/SimpleApp/Header.tsx b/projects/app/src/pages/app/detail/components/SimpleApp/Header.tsx
@@ -29,7 +29,7 @@ import {
 } from './useSnapshots';
 import PublishHistories from '../PublishHistoriesSlider';
 import { AppVersionSchemaType } from '@fastgpt/global/core/app/version';
-import { applyDiff } from '@/web/core/app/diff';
+import { getAppConfigByDiff } from '@/web/core/app/diff';
 
 const Header = ({
   forbiddenSaveSnapshot,
@@ -49,20 +49,13 @@ const Header = ({
   const { t } = useTranslation();
   const { isPc } = useSystem();
   const router = useRouter();
-  const { appId, onSaveApp, currentTab, appLatestVersion } = useContextSelector(
-    AppContext,
-    (v) => v
-  );
+  const appId = useContextSelector(AppContext, (v) => v.appId);
+  const onSaveApp = useContextSelector(AppContext, (v) => v.onSaveApp);
+  const currentTab = useContextSelector(AppContext, (v) => v.currentTab);
+  const appLatestVersion = useContextSelector(AppContext, (v) => v.appLatestVersion);
+
   const { lastAppListRouteType } = useSystemStore();
   const { allDatasets } = useDatasetStore();
-  const initialAppForm = useMemo(
-    () =>
-      appWorkflow2Form({
-        nodes: appLatestVersion?.nodes || [],
-        chatConfig: appLatestVersion?.chatConfig || {}
-      }),
-    [appLatestVersion]
-  );
 
   const { data: paths = [] } = useRequest2(() => getAppFolderPath(appId), {
     manual: false,
@@ -114,9 +107,18 @@ const Header = ({
   const [isShowHistories, { setTrue: setIsShowHistories, setFalse: closeHistories }] =
     useBoolean(false);
 
+  const initialAppForm = useMemo(
+    () =>
+      appWorkflow2Form({
+        nodes: appLatestVersion?.nodes || [],
+        chatConfig: appLatestVersion?.chatConfig || {}
+      }),
+    [appLatestVersion]
+  );
+
   const onSwitchTmpVersion = useCallback(
     (data: SimpleAppSnapshotType, customTitle: string) => {
-      const pastState = applyDiff(initialAppForm, data.diff);
+      const pastState = getAppConfigByDiff(initialAppForm, data.diff);
       setAppForm(pastState);
 
       // Remove multiple "copy-"
@@ -156,7 +158,7 @@ const Header = ({
   useDebounceEffect(
     () => {
       const savedSnapshot = past.find((snapshot) => snapshot.isSaved);
-      const pastState = applyDiff(initialAppForm, savedSnapshot?.diff);
+      const pastState = getAppConfigByDiff(initialAppForm, savedSnapshot?.diff);
       const val = compareSimpleAppSnapshot(pastState, appForm);
       setIsPublished(val);
     },

diff --git a/projects/app/src/pages/app/detail/components/SimpleApp/index.tsx b/projects/app/src/pages/app/detail/components/SimpleApp/index.tsx
@@ -49,7 +49,13 @@ const SimpleEdit = () => {
         saveSnapshot={saveSnapshot}
       />
       {currentTab === TabEnum.appEdit ? (
-        <Edit appForm={appForm} setAppForm={setAppForm} past={past} saveSnapshot={saveSnapshot} />
+        <Edit
+          appForm={appForm}
+          setAppForm={setAppForm}
+          past={past}
+          setPast={setPast}
+          saveSnapshot={saveSnapshot}
+        />
       ) : (
         <Box flex={'1 0 0'} h={0} mt={[4, 0]}>
           {currentTab === TabEnum.publish && <PublishChannel />}

diff --git a/projects/app/src/pages/app/detail/components/SimpleApp/useSnapshots.tsx b/projects/app/src/pages/app/detail/components/SimpleApp/useSnapshots.tsx
@@ -3,16 +3,19 @@ import { SetStateAction, useEffect, useRef } from 'react';
 import { formatTime2YMDHMS } from '@fastgpt/global/common/string/time';
 import { AppSimpleEditFormType } from '@fastgpt/global/core/app/type';
 import { isEqual } from 'lodash';
-import { applyDiff, createDiff } from '@/web/core/app/diff';
+import { getAppDiffConfig } from '@/web/core/app/diff';
 
 export type SimpleAppSnapshotType = {
   diff?: Record<string, any>;
   title: string;
   isSaved?: boolean;
   state?: AppSimpleEditFormType;
+
+  // old format
+  appForm?: AppSimpleEditFormType;
 };
 export type onSaveSnapshotFnType = (props: {
-  appForm: AppSimpleEditFormType;
+  appForm: AppSimpleEditFormType; // Current edited app form data
   title?: string;
   isSaved?: boolean;
 }) => Promise<boolean>;
@@ -58,7 +61,7 @@ export const compareSimpleAppSnapshot = (
 
 export const useSimpleAppSnapshots = (appId: string) => {
   const forbiddenSaveSnapshot = useRef(false);
-  const [past, setPast] = useLocalStorageState<SimpleAppSnapshotType[]>(`${appId}-past-simple`, {
+  const [past, setPast] = useLocalStorageState<SimpleAppSnapshotType[]>(`${appId}-past`, {
     defaultValue: []
   }) as [SimpleAppSnapshotType[], (value: SetStateAction<SimpleAppSnapshotType[]>) => void];
 
@@ -79,33 +82,36 @@ export const useSimpleAppSnapshots = (appId: string) => {
       return true;
     }
 
-    const initialState = past[past.length - 1].state;
-    if (!initialState) return false;
-
-    if (past.length > 0) {
-      const pastState = applyDiff(initialState, past[0].diff);
+    const lastPast = past[past.length - 1];
+    if (!lastPast?.state) return false;
 
-      const isPastEqual = compareSimpleAppSnapshot(pastState, appForm);
-      if (isPastEqual) return false;
-    }
+    // Get the diff between the current app form data and the initial state
+    const diff = getAppDiffConfig(lastPast.state, appForm);
 
-    const diff = createDiff(initialState, appForm);
+    // If the diff is the same as the previous snapshot, do not save
+    if (past[0].diff && isEqual(past[0].diff, diff)) return false;
 
-    setPast((past) => [
-      {
+    setPast((past) => {
+      const newPast = {
         diff,
         title: title || formatTime2YMDHMS(new Date()),
         isSaved
-      },
-      ...past.slice(0, 199)
-    ]);
+      };
+
+      if (past.length >= 100) {
+        return [newPast, ...past.slice(0, 98), lastPast];
+      }
+      return [newPast, ...past];
+    });
     return true;
   });
 
   // remove other app's snapshot
   useEffect(() => {
     const keys = Object.keys(localStorage);
-    const snapshotKeys = keys.filter((key) => key.endsWith('-past-simple'));
+    const snapshotKeys = keys.filter(
+      (key) => key.endsWith('-past') || key.endsWith('-past-simple')
+    );
     snapshotKeys.forEach((key) => {
       const keyAppId = key.split('-')[0];
       if (keyAppId !== appId) {
@@ -114,6 +120,20 @@ export const useSimpleAppSnapshots = (appId: string) => {
     });
   }, [appId]);
 
+  // 旧的编辑记录，直接重置到新的变量中
+  const [oldPast, setOldPast] = useLocalStorageState<SimpleAppSnapshotType[]>(
+    `${appId}-past-simple`,
+    {}
+  );
+  useEffect(() => {
+    if (oldPast && oldPast.length > 0) {
+      setPast(past);
+      setOldPast([]);
+      // refresh page
+      window.location.reload();
+    }
+  }, [oldPast]);
+
   return { forbiddenSaveSnapshot, past, setPast, saveSnapshot };
 };