Sandbox (#1610) (#1611)

.github/workflows/build-sandbox-image.yml

@@ -0,0 +1,84 @@
+name: Build fastgpt-sandbox images and copy image to docker hub
+on:
+  workflow_dispatch:
+  push:
+    paths:
+      - 'projects/sandbox/**'
+    tags:
+      - 'v*'
+jobs:
+  build-fastgpt-sandbox-images:
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Install Dependencies
+        run: |
+          sudo apt update && sudo apt install -y nodejs npm
+      - name: Set up QEMU (optional)
+        uses: docker/setup-qemu-action@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          driver-opts: network=host
+      - name: Cache Docker layers
+        uses: actions/cache@v2
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GH_PAT }}
+      - name: Set DOCKER_REPO_TAGGED based on branch or tag
+        run: |
+          if [[ "${{ github.ref_name }}" == "main" ]]; then
+            echo "DOCKER_REPO_TAGGED=ghcr.io/${{ github.repository_owner }}/fastgpt-sandbox:latest" >> $GITHUB_ENV
+          else
+            echo "DOCKER_REPO_TAGGED=ghcr.io/${{ github.repository_owner }}/fastgpt-sandbox:${{ github.ref_name }}" >> $GITHUB_ENV
+          fi
+      - name: Build and publish image for main branch or tag push event
+        env:
+          DOCKER_REPO_TAGGED: ${{ env.DOCKER_REPO_TAGGED }}
+        run: |
+          docker buildx build \
+          -f projects/sandbox/Dockerfile \
+          --platform linux/amd64,linux/arm64 \
+          --label "org.opencontainers.image.source=https://github.com/${{ github.repository_owner }}/fastgpt-sandbox" \
+          --label "org.opencontainers.image.description=fastgpt-sandbox image" \
+          --push \
+          --cache-from=type=local,src=/tmp/.buildx-cache \
+          --cache-to=type=local,dest=/tmp/.buildx-cache \
+          -t ${DOCKER_REPO_TAGGED} \
+          .
+  push-to-ali-hub:
+    needs: build-fastgpt-sandbox-images
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Login to Ali Hub
+        uses: docker/login-action@v2
+        with:
+          registry: registry.cn-hangzhou.aliyuncs.com
+          username: ${{ secrets.ALI_HUB_USERNAME }}
+          password: ${{ secrets.ALI_HUB_PASSWORD }}
+      - name: Set DOCKER_REPO_TAGGED based on branch or tag
+        run: |
+          if [[ "${{ github.ref_name }}" == "main" ]]; then
+            echo "IMAGE_TAG=latest" >> $GITHUB_ENV
+          else
+            echo "IMAGE_TAG=${{ github.ref_name }}" >> $GITHUB_ENV
+          fi
+      - name: Pull image from GitHub Container Registry
+        run: docker pull ghcr.io/${{ github.repository_owner }}/fastgpt-sandbox:${{env.IMAGE_TAG}}
+      - name: Tag image with Docker Hub repository name and version tag
+        run: docker tag ghcr.io/${{ github.repository_owner }}/fastgpt-sandbox:${{env.IMAGE_TAG}} ${{ secrets.ALI_IMAGE_NAME }}/fastgpt-sandbox:${{env.IMAGE_TAG}}
+      - name: Push image to Docker Hub
+        run: docker push ${{ secrets.ALI_IMAGE_NAME }}/fastgpt-sandbox:${{env.IMAGE_TAG}}

.github/workflows/fastgpt-image.yml

@@ -49,15 +49,14 @@ jobs:
           DOCKER_REPO_TAGGED: ${{ env.DOCKER_REPO_TAGGED }}
         run: |
           docker buildx build \
-          --build-arg name=app \
+          -f projects/app/Dockerfile \
           --platform linux/amd64,linux/arm64 \
           --label "org.opencontainers.image.source=https://github.com/${{ github.repository_owner }}/FastGPT" \
           --label "org.opencontainers.image.description=fastgpt image" \
           --push \
           --cache-from=type=local,src=/tmp/.buildx-cache \
           --cache-to=type=local,dest=/tmp/.buildx-cache \
           -t ${DOCKER_REPO_TAGGED} \
-          -f Dockerfile \
           .
   push-to-docker-hub:
     needs: build-fastgpt-images

.github/workflows/preview-image.yml

@@ -44,15 +44,14 @@ jobs:
           DOCKER_REPO_TAGGED: ${{ env.DOCKER_REPO_TAGGED }}
         run: |
           docker buildx build \
-          --build-arg name=app \
-          --label "org.opencontainers.image.source=  https://github.com/  ${{ github.repository_owner }}/FastGPT" \
-          --label "org.opencontainers.image.description=fastgpt-pr image" \
-          --label "org.opencontainers.image.licenses=Apache" \
+          -f projects/app/Dockerfile \
+          --label "org.opencontainers.image.source=https://github.com/${{ github.repository_owner }}/FastGPT" \
+          --label "org.opencontainers.image.description=fastgpt-pr imae" \
+          --label "org.opencontainers.image.licenses=Apache" \g
           --push \
           --cache-from=type=local,src=/tmp/.buildx-cache \
           --cache-to=type=local,dest=/tmp/.buildx-cache \
           -t ${DOCKER_REPO_TAGGED} \
-          -f Dockerfile \
           .
 
   helm-check:

Makefile

@@ -0,0 +1,25 @@
+# 定义默认变量
+proxy=null
+image=null
+
+# 定义目标
+.PHONY: build
+
+# 检查 target 是否定义
+ifndef name
+$(error name is not defined)
+endif
+
+filePath=./projects/$(name)/Dockerfile
+
+dev:
+	pnpm --prefix ./projects/$(name) dev
+
+build:
+ifeq ($(proxy), taobao)
+	docker build -f $(filePath) -t $(image) . --build-arg proxy=taobao 
+else ifeq ($(proxy), clash)
+	docker build -f $(filePath) -t $(image) . --network host --build-arg HTTP_PROXY=http://127.0.0.1:7890 --build-arg HTTPS_PROXY=http://127.0.0.1:7890
+else
+	docker build -f $(filePath) -t $(image) .
+endif

dev.md

@@ -1,17 +1,40 @@
-# 打包命令
+## Premise
+
+Since FastGPT is managed in the same way as monorepo, it is recommended to install 'make' first during development.
+
+monorepo Project Name:
+
+- app: main project
+-......
+
+## Dev
 
 ```sh
-# Build image, not proxy
-docker build -t registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.4.7 --build-arg name=app .
+# Give automatic script code execution permission (on non-Linux systems, you can manually execute the postinstall.sh file content)
+chmod -R +x ./scripts/
+# Executing under the code root directory installs all dependencies within the root package, projects, and packages
+pnpm i
+
+# Not make cmd
+cd projects/app
+pnpm dev
 
-# build image with proxy
-docker build -t registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.4.7 --build-arg name=app --build-arg proxy=taobao .
+# Make cmd
+make dev name=app
 ```
 
-# Pg 常用索引
 
-```sql
-CREATE INDEX IF NOT EXISTS modelData_dataset_id_index ON modeldata (dataset_id);
-CREATE INDEX IF NOT EXISTS modelData_collection_id_index ON modeldata (collection_id);
-CREATE INDEX IF NOT EXISTS modelData_teamId_index ON modeldata (team_id);
-```
+## Build
+
+```sh
+# Docker cmd: Build image, not proxy
+docker build -f ./projects/app/Dockerfile -t registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.8.1 . --build-arg name=app
+# Make cmd: Build image, not proxy
+make build name=app image=registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.8.1
+
+# Docker cmd: Build image with proxy
+docker build -f ./projects/app/Dockerfile -t registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.8.1 . --build-arg name=app --build-arg proxy=taobao
+# Make cmd: Build image with proxy
+make build name=app image=registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.8.1 proxy=taobao
+```
+

docSite/content/docs/development/intro.md

@@ -16,8 +16,9 @@ weight: 705
 
 - [Git](http://git-scm.com/)
 - [Docker](https://www.docker.com/)（构建镜像）
-- [Node.js v18.x (不推荐最新的，可能有兼容问题)](http://nodejs.org)
-- [pnpm](https://pnpm.io/) 版本 8.x.x 
+- [Node.js v18.17 / v20.x](http://nodejs.org)
+- [pnpm](https://pnpm.io/) 版本 8.6.0 (目前官方的开发环境)
+- make命令: 根据不同平台，百度安装 (官方是GNU Make 4.3)
 
 ## 开始本地开发
 
@@ -72,24 +73,34 @@ Mongo 数据库需要注意，需要注意在连接地址中增加 `directConnec
 
 ### 5. 运行
 
+可参考项目根目录下的 `dev.md`
+
 ```bash
 # 给自动化脚本代码执行权限(非 linux 系统, 可以手动执行里面的 postinstall.sh 文件内容)
 chmod -R +x ./scripts/
 # 代码根目录下执行，会安装根 package、projects 和 packages 内所有依赖
 pnpm i
-# 切换到应用目录
-cd projects/app 
-# 开发模式运行
+
+# 非 Make 运行
+cd projects/app
 pnpm dev
+
+# Make 运行
+make dev name=app
 ```
 
 ### 6. 部署打包
 
 ```bash
-# 根目录下执行
-docker build -t dockername/fastgpt:tag --build-arg name=app .
-# 使用代理
-docker build -t dockername/fastgpt:tag --build-arg name=app --build-arg proxy=taobao .
+# Docker cmd: Build image, not proxy
+docker build -f ./projects/app/Dockerfile -t registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.8.1 . --build-arg name=app
+# Make cmd: Build image, not proxy
+make build name=app image=registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.8.1
+
+# Docker cmd: Build image with proxy
+docker build -f ./projects/app/Dockerfile -t registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.8.1 . --build-arg name=app --build-arg proxy=taobao
+# Make cmd: Build image with proxy
+make build name=app image=registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.8.1 proxy=taobao
 ```
 
 ## 提交代码至开源仓库
@@ -101,21 +112,21 @@ docker build -t dockername/fastgpt:tag --build-arg name=app --build-arg proxy=ta
 如果遇到问题，比如合并冲突或不知道如何打开拉取请求，请查看 GitHub 的[拉取请求教程](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests)，了解如何解决合并冲突和其他问题。一旦您的 PR 被合并，您将自豪地被列为[贡献者表](https://github.com/labring/FastGPT/graphs/contributors)中的一员。
 
 
-
 ## QA
 
 ### 本地数据库无法连接
 
 1. 如果你是连接远程的数据库，先检查对应的端口是否开放。
 2. 如果是本地运行的数据库，可尝试`host`改成`localhost`或`127.0.0.1`
+3. 本地连接远程的 Mongo，需要增加 `directConnection=true` 参数，才能连接上副本集的数据库。
+4. mongo使用`mongocompass`客户端进行连接测试和可视化管理。
+5. pg使用`navicat`进行连接和管理。
 
 ### sh ./scripts/postinstall.sh 没权限
 
 FastGPT 在`pnpm i`后会执行`postinstall`脚本，用于自动生成`ChakraUI`的`Type`。如果没有权限，可以先执行`chmod -R +x ./scripts/`，再执行`pnpm i`。
 
-### 长时间运行后崩溃
-
-似乎是由于 tiktoken 库的开发环境问题，生产环境中未遇到，暂时可忽略。
+仍不可行的话，可以手动执行`./scripts/postinstall.sh`里的内容。
 
 ### TypeError: Cannot read properties of null (reading 'useMemo' )
 

packages/global/core/workflow/constants.ts

@@ -108,7 +108,11 @@ export enum NodeInputKeyEnum {
   ifElseList = 'ifElseList',
 
   // variable update
-  updateList = 'updateList'
+  updateList = 'updateList',
+
+  // code
+  code = 'code',
+  codeType = 'codeType' // js|py
 }
 
 export enum NodeOutputKeyEnum {
@@ -121,6 +125,7 @@ export enum NodeOutputKeyEnum {
   error = 'error',
   text = 'system_text',
   addOutputParam = 'system_addOutputParam',
+  rawResponse = 'system_rawResponse',
 
   // dataset
   datasetQuoteQA = 'quoteQA',

packages/global/core/workflow/node/constant.ts

@@ -113,7 +113,8 @@ export enum FlowNodeTypeEnum {
   stopTool = 'stopTool',
   lafModule = 'lafModule',
   ifElseNode = 'ifElseNode',
-  variableUpdate = 'variableUpdate'
+  variableUpdate = 'variableUpdate',
+  code = 'code'
 }
 
 export const EDGE_TYPE = 'default';

packages/global/core/workflow/runtime/type.d.ts

@@ -31,6 +31,8 @@ export type DispatchNodeResponseType = {
   runningTime?: number;
   query?: string;
   textOutput?: string;
+  customInputs?: Record<string, any>;
+  customOutputs?: Record<string, any>;
 
   // bill
   tokens?: number;

packages/global/core/workflow/template/constants.ts

@@ -22,6 +22,7 @@ import type { FlowNodeTemplateType } from '../type';
 import { LafModule } from './system/laf';
 import { IfElseNode } from './system/ifElse/index';
 import { VariableUpdateNode } from './system/variableUpdate';
+import { CodeNode } from './system/sandbox';
 
 /* app flow module templates */
 export const appSystemModuleTemplates: FlowNodeTemplateType[] = [
@@ -40,7 +41,8 @@ export const appSystemModuleTemplates: FlowNodeTemplateType[] = [
   AiQueryExtension,
   LafModule,
   IfElseNode,
-  VariableUpdateNode
+  VariableUpdateNode,
+  CodeNode
 ];
 /* plugin flow module templates */
 export const pluginSystemModuleTemplates: FlowNodeTemplateType[] = [
@@ -59,7 +61,8 @@ export const pluginSystemModuleTemplates: FlowNodeTemplateType[] = [
   AiQueryExtension,
   LafModule,
   IfElseNode,
-  VariableUpdateNode
+  VariableUpdateNode,
+  CodeNode
 ];
 
 /* all module */
@@ -84,5 +87,6 @@ export const moduleTemplatesFlat: FlowNodeTemplateType[] = [
   AiQueryExtension,
   LafModule,
   IfElseNode,
-  VariableUpdateNode
+  VariableUpdateNode,
+  CodeNode
 ];

packages/global/core/workflow/template/system/sandbox/constants.ts

@@ -0,0 +1,7 @@
+export const JS_TEMPLATE = `function main({data1, data2}){
+    
+    return {
+        result: data1,
+        data2
+    }
+}`;