perf: tiktoken count (#1507)

* perf: tiktoken count

* fix: rerank histories

* fix: rerank histories

* update npmrc

.npmrc

@@ -0,0 +1,2 @@
+public-hoist-pattern[]=*tiktoken*
+public-hoist-pattern[]=*react*

Dockerfile

@@ -11,7 +11,7 @@ RUN apk add --no-cache libc6-compat && npm install -g [email protected]
 RUN [ -z "$proxy" ] || pnpm config set registry https://registry.npmmirror.com
 
 # copy packages and one project
-COPY pnpm-lock.yaml pnpm-workspace.yaml ./
+COPY pnpm-lock.yaml pnpm-workspace.yaml .npmrc ./
 COPY ./packages ./packages
 COPY ./projects/$name/package.json ./projects/$name/package.json
 
@@ -27,7 +27,7 @@ ARG name
 ARG proxy
 
 # copy common node_modules and one project node_modules
-COPY package.json pnpm-workspace.yaml ./
+COPY package.json pnpm-workspace.yaml .npmrc ./
 COPY --from=mainDeps /app/node_modules ./node_modules
 COPY --from=mainDeps /app/packages ./packages
 COPY ./projects/$name ./projects/$name
@@ -64,6 +64,11 @@ COPY --from=builder --chown=nextjs:nodejs /app/projects/$name/.next/static /app/
 COPY --from=builder --chown=nextjs:nodejs /app/projects/$name/.next/server/chunks /app/projects/$name/.next/server/chunks
 # copy worker
 COPY --from=builder --chown=nextjs:nodejs /app/projects/$name/.next/server/worker /app/projects/$name/.next/server/worker
+
+# copy tiktoken but not copy ./node_modules/tiktoken/encoders
+COPY --from=mainDeps /app/node_modules/tiktoken ./node_modules/tiktoken
+RUN rm -rf ./node_modules/tiktoken/encoders
+
 # copy package.json to version file
 COPY --from=builder /app/projects/$name/package.json ./package.json 
 # copy config

packages/service/common/string/tiktoken/index.ts

@@ -47,41 +47,45 @@ export const countGptMessagesTokens = (
   tools?: ChatCompletionTool[],
   functionCall?: ChatCompletionCreateParams.Function[]
 ) => {
-  return new Promise<number>((resolve) => {
-    const start = Date.now();
-
-    const { worker, callbackMap } = getTiktokenWorker();
-
-    const id = getNanoid();
-
-    const timer = setTimeout(() => {
-      console.log('Count token Time out');
-      resolve(
-        messages.reduce((sum, item) => {
-          if (item.content) {
-            return sum + item.content.length * 0.5;
-          }
-          return sum;
-        }, 0)
-      );
-      delete callbackMap[id];
-    }, 60000);
-
-    callbackMap[id] = (data) => {
-      // 检测是否有内存泄漏
-      addLog.info(`Count token time: ${Date.now() - start}, token: ${data}`);
-      // console.log(process.memoryUsage());
-
-      resolve(data);
-      clearTimeout(timer);
-    };
-
-    worker.postMessage({
-      id,
-      messages,
-      tools,
-      functionCall
-    });
+  return new Promise<number>(async (resolve) => {
+    try {
+      const start = Date.now();
+
+      const { worker, callbackMap } = getTiktokenWorker();
+
+      const id = getNanoid();
+
+      const timer = setTimeout(() => {
+        console.log('Count token Time out');
+        resolve(
+          messages.reduce((sum, item) => {
+            if (item.content) {
+              return sum + item.content.length * 0.5;
+            }
+            return sum;
+          }, 0)
+        );
+        delete callbackMap[id];
+      }, 60000);
+
+      callbackMap[id] = (data) => {
+        // 检测是否有内存泄漏
+        addLog.info(`Count token time: ${Date.now() - start}, token: ${data}`);
+        // console.log(process.memoryUsage());
+
+        resolve(data);
+        clearTimeout(timer);
+      };
+
+      worker.postMessage({
+        id,
+        messages,
+        tools,
+        functionCall
+      });
+    } catch (error) {
+      resolve(0);
+    }
   });
 };
 

packages/service/core/dataset/search/utils.ts

@@ -2,6 +2,7 @@ import { LLMModelItemType } from '@fastgpt/global/core/ai/model.d';
 import { queryExtension } from '../../ai/functions/queryExtension';
 import { ChatItemType } from '@fastgpt/global/core/chat/type';
 import { hashStr } from '@fastgpt/global/common/string/tools';
+import { chatValue2RuntimePrompt } from '@fastgpt/global/core/chat/adapt';
 
 export const datasetSearchQueryExtension = async ({
   query,
@@ -33,11 +34,11 @@ export const datasetSearchQueryExtension = async ({
       histories.length > 0
         ? `${histories
             .map((item) => {
-              return `${item.obj}: ${item.value}`;
+              return `${item.obj}: ${chatValue2RuntimePrompt(item.value).text}`;
             })
             .join('\n')}
-  Human: ${query}
-  `
+Human: ${query}
+`
         : query;
 
     /* if query already extension, direct parse */

packages/service/package.json

@@ -13,7 +13,6 @@
     "decompress": "^4.2.1",
     "domino-ext": "^2.1.4",
     "encoding": "^0.1.13",
-    "fastgpt-js-tiktoken": "^1.0.12",
     "file-type": "^19.0.0",
     "iconv-lite": "^0.6.3",
     "joplin-turndown-plugin-gfm": "^1.0.12",
@@ -29,6 +28,7 @@
     "papaparse": "5.4.1",
     "pdfjs-dist": "4.0.269",
     "pg": "^8.10.0",
+    "tiktoken": "^1.0.15",
     "tunnel": "^0.0.6",
     "turndown": "^7.1.2"
   },

packages/service/worker/tiktoken/countGptMessagesTokens.ts

@@ -1,5 +1,5 @@
 /* Only the token of gpt-3.5-turbo is used */
-import { Tiktoken } from 'fastgpt-js-tiktoken/lite';
+import { Tiktoken } from 'tiktoken/lite';
 import cl100k_base from './cl100k_base.json';
 import {
   ChatCompletionMessageParam,
@@ -10,7 +10,7 @@ import {
 import { ChatCompletionRequestMessageRoleEnum } from '@fastgpt/global/core/ai/constants';
 import { parentPort } from 'worker_threads';
 
-const enc = new Tiktoken(cl100k_base);
+const enc = new Tiktoken(cl100k_base.bpe_ranks, cl100k_base.special_tokens, cl100k_base.pat_str);
 
 /* count messages tokens */
 parentPort?.on(

projects/app/next.config.js

@@ -36,6 +36,10 @@ const nextConfig = {
       unknownContextCritical: false
     };
 
+    if (!config.externals) {
+      config.externals = [];
+    }
+
     if (isServer) {
       config.externals.push('worker_threads');
 
@@ -73,11 +77,13 @@ const nextConfig = {
           fs: false
         }
       };
-      if (!config.externals) {
-        config.externals = [];
-      }
     }
 
+    config.experiments = {
+      asyncWebAssembly: true,
+      layers: true
+    };
+
     return config;
   },
   transpilePackages: ['@fastgpt/*', 'ahooks', '@chakra-ui/*', 'react'],