perf: vision model url

labring · Jul 16, 2024 · 259bc0d · 259bc0d
1 parent 5b5d231
commit 259bc0d
Show file tree

Hide file tree

Showing 13 changed files with 76 additions and 269 deletions.
diff --git a/docSite/content/zh-cn/docs/development/upgrading/487.md b/docSite/content/zh-cn/docs/development/upgrading/487.md
@@ -24,4 +24,6 @@ weight: 817
 2. 新增 - 应用搜索
 3. 优化 - 对话框代码
 4. 优化 - 升级 Dockerfile node 和 pnpm 版本
-5. 修复 - 简易模式无法变更全局变量
+5. 优化 - local 域名部署，也可以正常使用 vision 模式
+6. 修复 - 简易模式无法变更全局变量
+7. 修复 - gpt4o 无法同时使用工具和图片
diff --git a/packages/global/core/chat/constants.ts b/packages/global/core/chat/constants.ts
@@ -56,7 +56,4 @@ export enum ChatStatusEnum {
   finish = 'finish'
 }
 
-export const IMG_BLOCK_KEY = 'img-block';
-export const FILE_BLOCK_KEY = 'file-block';
-
 export const MARKDOWN_QUOTE_SIGN = 'QUOTE SIGN';
diff --git a/packages/service/common/api/serverRequest.ts b/packages/service/common/api/serverRequest.ts
@@ -63,6 +63,7 @@ const instance = axios.create({
     'Cache-Control': 'no-cache'
   }
 });
+export const serverRequestBaseUrl = `http://${SERVICE_LOCAL_HOST}`;
 
 /* 请求拦截 */
 instance.interceptors.request.use(requestStart, (err) => Promise.reject(err));
@@ -79,7 +80,7 @@ export function request(url: string, data: any, config: ConfigType, method: Meth
 
   return instance
     .request({
-      baseURL: `http://${SERVICE_LOCAL_HOST}`,
+      baseURL: serverRequestBaseUrl,
       url,
       method,
       data: ['POST', 'PUT'].includes(method) ? data : null,

diff --git a/packages/service/common/mongo/index.ts b/packages/service/common/mongo/index.ts
@@ -64,6 +64,7 @@ export const getMongoModel = <T>(name: string, schema: mongoose.Schema) => {
   addCommonMiddleware(schema);
 
   const model = connectionMongo.model<T>(name, schema);
+
   try {
     // model.createIndexes({ background: true });
     model.syncIndexes({ background: true });

diff --git a/packages/service/core/chat/utils.ts b/packages/service/core/chat/utils.ts
@@ -1,4 +1,3 @@
-import { IMG_BLOCK_KEY } from '@fastgpt/global/core/chat/constants';
 import { countGptMessagesTokens } from '../../common/string/tiktoken/index';
 import type {
   ChatCompletionContentPart,
@@ -7,6 +6,8 @@ import type {
 import axios from 'axios';
 import { ChatCompletionRequestMessageRoleEnum } from '@fastgpt/global/core/ai/constants';
 import { guessBase64ImageType } from '../../common/file/utils';
+import { serverRequestBaseUrl } from '../../common/api/serverRequest';
+import { cloneDeep } from 'lodash';
 
 /* slice chat context by tokens */
 const filterEmptyMessages = (messages: ChatCompletionMessageParam[]) => {
@@ -120,140 +121,64 @@ export const formatGPTMessagesInRequestBefore = (messages: ChatCompletionMessage
     .filter(Boolean) as ChatCompletionMessageParam[];
 };
 
-/**
-    string to vision model. Follow the markdown code block rule for interception:
-
-    @rule:
-    ```img-block
-        {src:""}
-        {src:""}
-    ```
-    ```file-block
-        {name:"",src:""},
-        {name:"",src:""}
-    ```
-    @example:
-        What’s in this image?
-        ```img-block
-            {src:"https://1.png"}
-        ```
-    @return 
-        [
-            { type: 'text', text: 'What’s in this image?' },
-            {
-              type: 'image_url',
-              image_url: {
-                url: 'https://1.png'
-              }
-            }
-        ]
- */
-export async function formatStr2ChatContent(str: string) {
-  const content: ChatCompletionContentPart[] = [];
-  let lastIndex = 0;
-  const regex = new RegExp(`\`\`\`(${IMG_BLOCK_KEY})\\n([\\s\\S]*?)\`\`\``, 'g');
-
-  const imgKey: 'image_url' = 'image_url';
-
-  let match;
-
-  while ((match = regex.exec(str)) !== null) {
-    // add previous text
-    if (match.index > lastIndex) {
-      const text = str.substring(lastIndex, match.index).trim();
-      if (text) {
-        content.push({ type: 'text', text });
-      }
-    }
-
-    const blockType = match[1].trim();
-
-    if (blockType === IMG_BLOCK_KEY) {
-      const blockContentLines = match[2].trim().split('\n');
-      const jsonLines = blockContentLines.map((item) => {
-        try {
-          return JSON.parse(item) as { src: string };
-        } catch (error) {
-          return { src: '' };
-        }
-      });
-
-      for (const item of jsonLines) {
-        if (!item.src) throw new Error("image block's content error");
-      }
-
-      content.push(
-        ...jsonLines.map((item) => ({
-          type: imgKey,
-          image_url: {
-            url: item.src
-          }
-        }))
-      );
-    }
-
-    lastIndex = regex.lastIndex;
-  }
-
-  // add remaining text
-  if (lastIndex < str.length) {
-    const remainingText = str.substring(lastIndex).trim();
-    if (remainingText) {
-      content.push({ type: 'text', text: remainingText });
-    }
-  }
-
-  // Continuous text type content, if type=text, merge them
-  for (let i = 0; i < content.length - 1; i++) {
-    const currentContent = content[i];
-    const nextContent = content[i + 1];
-    if (currentContent.type === 'text' && nextContent.type === 'text') {
-      currentContent.text += nextContent.text;
-      content.splice(i + 1, 1);
-      i--;
-    }
-  }
-
-  if (content.length === 1 && content[0].type === 'text') {
-    return content[0].text;
-  }
-
-  if (!content) return null;
-  // load img to base64
-  for await (const item of content) {
-    if (item.type === imgKey && item[imgKey]?.url) {
-      const response = await axios.get(item[imgKey].url, {
-        responseType: 'arraybuffer'
-      });
-      const base64 = Buffer.from(response.data).toString('base64');
-      item[imgKey].url = `data:${response.headers['content-type']};base64,${base64}`;
-    }
-  }
-
-  return content ? content : null;
-}
-
+/* Load user chat content.
+  Img: to base 64
+*/
 export const loadChatImgToBase64 = async (content: string | ChatCompletionContentPart[]) => {
   if (typeof content === 'string') {
     return content;
   }
+
   return Promise.all(
     content.map(async (item) => {
       if (item.type === 'text') return item;
+
+      if (!item.image_url.url) return item;
+
       /* 
         1. From db: Get it from db
         2. From web: Not update
       */
-      const response = await axios.get(item.image_url.url, {
-        responseType: 'arraybuffer'
-      });
-      const base64 = Buffer.from(response.data).toString('base64');
-      let imageType = response.headers['content-type'];
-      if (imageType === undefined) {
-        imageType = guessBase64ImageType(base64);
+      if (item.image_url.url.startsWith('/')) {
+        const response = await axios.get(item.image_url.url, {
+          baseURL: serverRequestBaseUrl,
+          responseType: 'arraybuffer'
+        });
+        const base64 = Buffer.from(response.data).toString('base64');
+        let imageType = response.headers['content-type'];
+        if (imageType === undefined) {
+          imageType = guessBase64ImageType(base64);
+        }
+        return {
+          ...item,
+          image_url: {
+            ...item.image_url,
+            url: `data:${imageType};base64,${base64}`
+          }
+        };
       }
-      item.image_url.url = `data:${imageType};base64,${base64}`;
+
       return item;
     })
   );
 };
+export const loadRequestMessages = async (messages: ChatCompletionMessageParam[]) => {
+  if (messages.length === 0) {
+    return Promise.reject('core.chat.error.Messages empty');
+  }
+
+  const loadMessages = await Promise.all(
+    messages.map(async (item) => {
+      if (item.role === ChatCompletionRequestMessageRoleEnum.User) {
+        return {
+          ...item,
+          content: await loadChatImgToBase64(item.content)
+        };
+      } else {
+        return item;
+      }
+    })
+  );
+
+  return loadMessages;
+};
diff --git a/packages/service/core/workflow/dispatch/agent/runTool/functionCall.ts b/packages/service/core/workflow/dispatch/agent/runTool/functionCall.ts
@@ -1,6 +1,6 @@
 import { LLMModelItemType } from '@fastgpt/global/core/ai/model.d';
 import { getAIApi } from '../../../../ai/config';
-import { filterGPTMessageByMaxTokens } from '../../../../chat/utils';
+import { filterGPTMessageByMaxTokens, loadRequestMessages } from '../../../../chat/utils';
 import {
   ChatCompletion,
   StreamChatType,
@@ -88,6 +88,7 @@ export const runToolWithFunctionCall = async (
     }
     return item;
   });
+  const requestMessages = await loadRequestMessages(formativeMessages);
 
   /* Run llm */
   const ai = getAIApi({
@@ -99,7 +100,7 @@ export const runToolWithFunctionCall = async (
       model: toolModel.model,
       temperature: 0,
       stream,
-      messages: formativeMessages,
+      messages: requestMessages,
       functions,
       function_call: 'auto'
     },

diff --git a/packages/service/core/workflow/dispatch/agent/runTool/index.ts b/packages/service/core/workflow/dispatch/agent/runTool/index.ts
@@ -12,6 +12,7 @@ import { ChatItemType } from '@fastgpt/global/core/chat/type';
 import { ChatRoleEnum } from '@fastgpt/global/core/chat/constants';
 import {
   GPTMessages2Chats,
+  chatValue2RuntimePrompt,
   chats2GPTMessages,
   getSystemPrompt,
   runtimePrompt2ChatsValue
@@ -29,10 +30,11 @@ type Response = DispatchNodeResultType<{
 
 export const dispatchRunTools = async (props: DispatchToolModuleProps): Promise<Response> => {
   const {
-    node: { nodeId, name, outputs },
+    node: { nodeId, name },
     runtimeNodes,
     runtimeEdges,
     histories,
+    query,
     params: { model, systemPrompt, userChatInput, history = 6 }
   } = props;
 
@@ -65,7 +67,7 @@ export const dispatchRunTools = async (props: DispatchToolModuleProps): Promise<
       obj: ChatRoleEnum.Human,
       value: runtimePrompt2ChatsValue({
         text: userChatInput,
-        files: []
+        files: chatValue2RuntimePrompt(query).files
       })
     }
   ];

diff --git a/packages/service/core/workflow/dispatch/agent/runTool/promptCall.ts b/packages/service/core/workflow/dispatch/agent/runTool/promptCall.ts
@@ -1,6 +1,6 @@
 import { LLMModelItemType } from '@fastgpt/global/core/ai/model.d';
 import { getAIApi } from '../../../../ai/config';
-import { filterGPTMessageByMaxTokens } from '../../../../chat/utils';
+import { filterGPTMessageByMaxTokens, loadRequestMessages } from '../../../../chat/utils';
 import {
   ChatCompletion,
   StreamChatType,
@@ -87,6 +87,8 @@ export const runToolWithPromptCall = async (
     messages,
     maxTokens: toolModel.maxContext - 500 // filter token. not response maxToken
   });
+  const requestMessages = await loadRequestMessages(filterMessages);
+
   // console.log(JSON.stringify(filterMessages, null, 2));
   /* Run llm */
   const ai = getAIApi({
@@ -98,7 +100,7 @@ export const runToolWithPromptCall = async (
       model: toolModel.model,
       temperature: 0,
       stream,
-      messages: filterMessages
+      messages: requestMessages
     },
     {
       headers: {

diff --git a/packages/service/core/workflow/dispatch/agent/runTool/toolChoice.ts b/packages/service/core/workflow/dispatch/agent/runTool/toolChoice.ts
@@ -1,6 +1,6 @@
 import { LLMModelItemType } from '@fastgpt/global/core/ai/model.d';
 import { getAIApi } from '../../../../ai/config';
-import { filterGPTMessageByMaxTokens } from '../../../../chat/utils';
+import { filterGPTMessageByMaxTokens, loadRequestMessages } from '../../../../chat/utils';
 import {
   ChatCompletion,
   ChatCompletionMessageToolCall,
@@ -99,14 +99,16 @@ export const runToolWithToolChoice = async (
     }
     return item;
   });
+  const requestMessages = await loadRequestMessages(formativeMessages);
+
   // console.log(
   //   JSON.stringify(
   //     {
   //       ...toolModel?.defaultConfig,
   //       model: toolModel.model,
   //       temperature: 0,
   //       stream,
-  //       messages: formativeMessages,
+  //       messages: requestMessages,
   //       tools,
   //       tool_choice: 'auto'
   //     },
@@ -124,7 +126,7 @@ export const runToolWithToolChoice = async (
       model: toolModel.model,
       temperature: 0,
       stream,
-      messages: formativeMessages,
+      messages: requestMessages,
       tools,
       tool_choice: 'auto'
     },

diff --git a/packages/service/core/workflow/dispatch/chat/oneapi.ts b/packages/service/core/workflow/dispatch/chat/oneapi.ts
@@ -2,7 +2,7 @@ import type { NextApiResponse } from 'next';
 import {
   filterGPTMessageByMaxTokens,
   formatGPTMessagesInRequestBefore,
-  loadChatImgToBase64
+  loadRequestMessages
 } from '../../../chat/utils';
 import type { ChatItemType, UserChatItemValueItemType } from '@fastgpt/global/core/chat/type.d';
 import { ChatRoleEnum } from '@fastgpt/global/core/chat/constants';
@@ -151,30 +151,15 @@ export const dispatchChatCompletion = async (props: ChatProps): Promise<ChatResp
     ...formatGPTMessagesInRequestBefore(filterMessages)
   ] as ChatCompletionMessageParam[];
 
-  if (concatMessages.length === 0) {
-    return Promise.reject('core.chat.error.Messages empty');
-  }
-
-  const loadMessages = await Promise.all(
-    concatMessages.map(async (item) => {
-      if (item.role === ChatCompletionRequestMessageRoleEnum.User) {
-        return {
-          ...item,
-          content: await loadChatImgToBase64(item.content)
-        };
-      } else {
-        return item;
-      }
-    })
-  );
+  const requestMessages = await loadRequestMessages(concatMessages);
 
   const requestBody = {
     ...modelConstantsData?.defaultConfig,
     model: modelConstantsData.model,
     temperature,
     max_tokens,
     stream,
-    messages: loadMessages
+    messages: requestMessages
   };
   const response = await ai.chat.completions.create(requestBody, {
     headers: {