From 5605f1a8922c5f408f37dfff6b3fc8e65d5acbb1 Mon Sep 17 00:00:00 2001 From: Archer <545436317@qq.com> Date: Fri, 5 Jul 2024 17:37:42 +0800 Subject: [PATCH] 4.8.6 fix (#1963) * feat: log store * fix: full text search match query * perf: mongo schema import, Avoid duplicate import --- .vscode/extensions.json | 5 -- .vscode/settings.json | 2 +- files/docker/docker-compose-milvus.yml | 1 + files/docker/docker-compose-pgvector.yml | 1 + files/docker/docker-compose-zilliz.yml | 1 + packages/global/tsconfig.json | 2 +- .../service/common/buffer/rawText/schema.ts | 9 +-- packages/service/common/buffer/tts/schema.ts | 6 +- packages/service/common/file/gridfs/schema.ts | 6 +- packages/service/common/file/image/schema.ts | 7 +- packages/service/common/mongo/index.ts | 72 +++++++++++++++++-- .../service/common/system/config/schema.ts | 9 +-- packages/service/common/system/log.ts | 71 +++++++----------- .../service/common/system/log/constant.ts | 10 +++ packages/service/common/system/log/schema.ts | 27 +++++++ packages/service/common/system/log/type.d.ts | 9 +++ .../service/common/system/timerLock/schema.ts | 6 +- packages/service/core/app/schema.ts | 19 ++--- packages/service/core/app/version/schema.ts | 10 +-- packages/service/core/chat/chatItemSchema.ts | 8 +-- packages/service/core/chat/chatSchema.ts | 6 +- .../service/core/chat/inputGuide/schema.ts | 10 +-- .../service/core/dataset/collection/schema.ts | 12 ++-- packages/service/core/dataset/data/schema.ts | 44 ++++++------ packages/service/core/dataset/schema.ts | 7 +- .../service/core/dataset/search/controller.ts | 4 +- .../service/core/dataset/training/schema.ts | 10 +-- .../core/workflow/dispatch/agent/extract.ts | 2 +- .../support/activity/promotion/schema.ts | 9 +-- packages/service/support/openapi/schema.ts | 6 +- packages/service/support/outLink/schema.ts | 7 +- packages/service/support/permission/schema.ts | 11 ++- packages/service/support/user/schema.ts | 6 +- .../support/user/team/teamMemberSchema.ts | 8 ++- .../service/support/user/team/teamSchema.ts | 5 +- .../support/user/team/teamTagsSchema.ts | 8 ++- packages/service/support/wallet/sub/schema.ts | 5 +- .../service/support/wallet/usage/schema.ts | 6 +- projects/app/.env.template | 6 +- 39 files changed, 252 insertions(+), 201 deletions(-) delete mode 100644 .vscode/extensions.json create mode 100644 packages/service/common/system/log/constant.ts create mode 100644 packages/service/common/system/log/schema.ts create mode 100644 packages/service/common/system/log/type.d.ts diff --git a/.vscode/extensions.json b/.vscode/extensions.json deleted file mode 100644 index 116d6852a68..00000000000 --- a/.vscode/extensions.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "recommendations": [ - "inlang.vs-code-extension" - ] -} \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json index f6f5e22aec1..9bd7803db8e 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -2,7 +2,7 @@ "editor.formatOnSave": true, "editor.mouseWheelZoom": true, "editor.defaultFormatter": "esbenp.prettier-vscode", - "prettier.prettierPath": "../node_modules/prettier", + "prettier.prettierPath": "node_modules/prettier", "typescript.tsdk": "node_modules/typescript/lib", "i18n-ally.localesPaths": [ "packages/web/i18n", diff --git a/files/docker/docker-compose-milvus.yml b/files/docker/docker-compose-milvus.yml index 1e6e2069f42..efce7016b81 100644 --- a/files/docker/docker-compose-milvus.yml +++ b/files/docker/docker-compose-milvus.yml @@ -156,6 +156,7 @@ services: - SANDBOX_URL=http://sandbox:3000 # 日志等级: debug, info, warn, error - LOG_LEVEL=info + - STORE_LOG_LEVEL=warn volumes: - ./config.json:/app/data/config.json diff --git a/files/docker/docker-compose-pgvector.yml b/files/docker/docker-compose-pgvector.yml index fb25835f2df..7f9c11370e6 100644 --- a/files/docker/docker-compose-pgvector.yml +++ b/files/docker/docker-compose-pgvector.yml @@ -113,6 +113,7 @@ services: - SANDBOX_URL=http://sandbox:3000 # 日志等级: debug, info, warn, error - LOG_LEVEL=info + - STORE_LOG_LEVEL=warn volumes: - ./config.json:/app/data/config.json diff --git a/files/docker/docker-compose-zilliz.yml b/files/docker/docker-compose-zilliz.yml index 99fab9e02dc..a94704b7dc6 100644 --- a/files/docker/docker-compose-zilliz.yml +++ b/files/docker/docker-compose-zilliz.yml @@ -94,6 +94,7 @@ services: - SANDBOX_URL=http://sandbox:3000 # 日志等级: debug, info, warn, error - LOG_LEVEL=info + - STORE_LOG_LEVEL=warn volumes: - ./config.json:/app/data/config.json diff --git a/packages/global/tsconfig.json b/packages/global/tsconfig.json index c7f84d24a2c..6f5dee2ff73 100644 --- a/packages/global/tsconfig.json +++ b/packages/global/tsconfig.json @@ -1,5 +1,5 @@ { - "extends":"../../tsconfig.json", + "extends": "../../tsconfig.json", "compilerOptions": { "baseUrl": "." }, diff --git a/packages/service/common/buffer/rawText/schema.ts b/packages/service/common/buffer/rawText/schema.ts index 64b37f5b0a4..43c4e408749 100644 --- a/packages/service/common/buffer/rawText/schema.ts +++ b/packages/service/common/buffer/rawText/schema.ts @@ -1,4 +1,4 @@ -import { connectionMongo, type Model } from '../../mongo'; +import { connectionMongo, getMongoModel, type Model } from '../../mongo'; const { Schema, model, models } = connectionMongo; import { RawTextBufferSchemaType } from './type'; @@ -28,6 +28,7 @@ try { console.log(error); } -export const MongoRawTextBuffer: Model = - models[collectionName] || model(collectionName, RawTextBufferSchema); -MongoRawTextBuffer.syncIndexes(); +export const MongoRawTextBuffer = getMongoModel( + collectionName, + RawTextBufferSchema +); diff --git a/packages/service/common/buffer/tts/schema.ts b/packages/service/common/buffer/tts/schema.ts index 1ffdadc5fcd..ab22f05ebc0 100644 --- a/packages/service/common/buffer/tts/schema.ts +++ b/packages/service/common/buffer/tts/schema.ts @@ -1,4 +1,4 @@ -import { connectionMongo, type Model } from '../../../common/mongo'; +import { connectionMongo, getMongoModel, type Model } from '../../../common/mongo'; const { Schema, model, models } = connectionMongo; import { TTSBufferSchemaType } from './type.d'; @@ -31,6 +31,4 @@ try { console.log(error); } -export const MongoTTSBuffer: Model = - models[collectionName] || model(collectionName, TTSBufferSchema); -MongoTTSBuffer.syncIndexes(); +export const MongoTTSBuffer = getMongoModel(collectionName, TTSBufferSchema); diff --git a/packages/service/common/file/gridfs/schema.ts b/packages/service/common/file/gridfs/schema.ts index 457447d7f31..0b72a80be06 100644 --- a/packages/service/common/file/gridfs/schema.ts +++ b/packages/service/common/file/gridfs/schema.ts @@ -1,4 +1,4 @@ -import { connectionMongo, type Model } from '../../mongo'; +import { connectionMongo, getMongoModel, type Model } from '../../mongo'; const { Schema, model, models } = connectionMongo; const FileSchema = new Schema({}); @@ -10,6 +10,4 @@ try { console.log(error); } -export const MongoFileSchema = models['dataset.files'] || model('dataset.files', FileSchema); - -MongoFileSchema.syncIndexes(); +export const MongoFileSchema = getMongoModel('dataset.files', FileSchema); diff --git a/packages/service/common/file/image/schema.ts b/packages/service/common/file/image/schema.ts index 8067ad9da1c..f48e50f01af 100644 --- a/packages/service/common/file/image/schema.ts +++ b/packages/service/common/file/image/schema.ts @@ -1,5 +1,5 @@ import { TeamCollectionName } from '@fastgpt/global/support/user/team/constant'; -import { connectionMongo, type Model } from '../../mongo'; +import { connectionMongo, getMongoModel, type Model } from '../../mongo'; import { MongoImageSchemaType } from '@fastgpt/global/common/file/image/type.d'; import { mongoImageTypeMap } from '@fastgpt/global/common/file/image/constants'; const { Schema, model, models } = connectionMongo; @@ -41,7 +41,4 @@ try { console.log(error); } -export const MongoImage: Model = - models['image'] || model('image', ImageSchema); - -MongoImage.syncIndexes(); +export const MongoImage = getMongoModel('image', ImageSchema); diff --git a/packages/service/common/mongo/index.ts b/packages/service/common/mongo/index.ts index 940370a257c..a31239106c5 100644 --- a/packages/service/common/mongo/index.ts +++ b/packages/service/common/mongo/index.ts @@ -1,7 +1,5 @@ -import mongoose from 'mongoose'; - -export default mongoose; -export * from 'mongoose'; +import { addLog } from '../../common/system/log'; +import mongoose, { Model } from 'mongoose'; export const connectionMongo = (() => { if (!global.mongodb) { @@ -11,4 +9,68 @@ export const connectionMongo = (() => { return global.mongodb; })(); -export const ReadPreference = mongoose.mongo.ReadPreference; +export default mongoose; +export * from 'mongoose'; + +const addCommonMiddleware = (schema: mongoose.Schema) => { + const operations = [ + /^find/, + 'save', + 'create', + /^update/, + /^delete/, + 'aggregate', + 'count', + 'countDocuments', + 'estimatedDocumentCount', + 'distinct', + 'insertMany' + ]; + + operations.forEach((op: any) => { + schema.pre(op, function (this: any, next) { + this._startTime = Date.now(); + this._query = this.getQuery ? this.getQuery() : null; + + next(); + }); + + schema.post(op, function (this: any, result: any, next) { + if (this._startTime) { + const duration = Date.now() - this._startTime; + + const warnLogData = { + query: this._query, + op, + duration + }; + + if (duration > 1000) { + addLog.warn(`Slow operation ${duration}ms`, warnLogData); + } else if (duration > 300) { + addLog.error(`Slow operation ${duration}ms`, warnLogData); + } + } + next(); + }); + }); + + return schema; +}; + +export const getMongoModel = (name: string, schema: mongoose.Schema) => { + if (connectionMongo.models[name]) return connectionMongo.models[name] as Model; + console.log('Load model======', name); + addCommonMiddleware(schema); + + const model = connectionMongo.model(name, schema); + try { + model.syncIndexes(); + } catch (error) { + addLog.error('Create index error', error); + } + + return model; +}; + +export const ReadPreference = connectionMongo.mongo.ReadPreference; diff --git a/packages/service/common/system/config/schema.ts b/packages/service/common/system/config/schema.ts index 2c5aab94bb7..57990c20343 100644 --- a/packages/service/common/system/config/schema.ts +++ b/packages/service/common/system/config/schema.ts @@ -1,5 +1,5 @@ import { SystemConfigsType } from '@fastgpt/global/common/system/config/type'; -import { connectionMongo, type Model } from '../../../common/mongo'; +import { connectionMongo, getMongoModel, type Model } from '../../../common/mongo'; import { SystemConfigsTypeMap } from '@fastgpt/global/common/system/config/constants'; const { Schema, model, models } = connectionMongo; @@ -27,6 +27,7 @@ try { console.log(error); } -export const MongoSystemConfigs: Model = - models[collectionName] || model(collectionName, systemConfigSchema); -MongoSystemConfigs.syncIndexes(); +export const MongoSystemConfigs = getMongoModel( + collectionName, + systemConfigSchema +); diff --git a/packages/service/common/system/log.ts b/packages/service/common/system/log.ts index 768f943f1c5..49bf208bfc2 100644 --- a/packages/service/common/system/log.ts +++ b/packages/service/common/system/log.ts @@ -1,13 +1,9 @@ import dayjs from 'dayjs'; import chalk from 'chalk'; -import { isProduction } from './constants'; +import { LogLevelEnum } from './log/constant'; +// import { MongoLog } from './log/schema'; +import connectionMongo from '../mongo/index'; -enum LogLevelEnum { - debug = 0, - info = 1, - warn = 2, - error = 3 -} const logMap = { [LogLevelEnum.debug]: { levelLog: chalk.green('[Debug]') @@ -23,23 +19,26 @@ const logMap = { } }; const envLogLevelMap: Record = { - debug: 0, - info: 1, - warn: 2, - error: 3 + debug: LogLevelEnum.debug, + info: LogLevelEnum.info, + warn: LogLevelEnum.warn, + error: LogLevelEnum.error }; -const logLevel = (() => { - if (!isProduction) return LogLevelEnum.debug; - const envLogLevel = (process.env.LOG_LEVEL || 'info').toLocaleLowerCase(); - if (!envLogLevel || envLogLevelMap[envLogLevel] === undefined) return LogLevelEnum.info; - return envLogLevelMap[envLogLevel]; +const { LOG_LEVEL, STORE_LOG_LEVEL } = (() => { + const LOG_LEVEL = (process.env.LOG_LEVEL || 'info').toLocaleLowerCase(); + const STORE_LOG_LEVEL = (process.env.STORE_LOG_LEVEL || '').toLocaleLowerCase(); + + return { + LOG_LEVEL: envLogLevelMap[LOG_LEVEL] || LogLevelEnum.info, + STORE_LOG_LEVEL: envLogLevelMap[STORE_LOG_LEVEL] ?? 99 + }; })(); /* add logger */ export const addLog = { log(level: LogLevelEnum, msg: string, obj: Record = {}) { - if (level < logLevel) return; + if (level < LOG_LEVEL) return; const stringifyObj = JSON.stringify(obj); const isEmpty = Object.keys(obj).length === 0; @@ -52,35 +51,15 @@ export const addLog = { level === LogLevelEnum.error && console.error(obj); - const lokiUrl = process.env.LOKI_LOG_URL as string; - if (!lokiUrl) return; - - try { - fetch(lokiUrl, { - method: 'POST', - headers: { - 'Content-type': 'application/json' - }, - body: JSON.stringify({ - streams: [ - { - stream: { - level - }, - values: [ - [ - `${Date.now() * 1000000}`, - JSON.stringify({ - message: msg, - ...obj - }) - ] - ] - } - ] - }) - }); - } catch (error) {} + // store + // if (level >= STORE_LOG_LEVEL && connectionMongo.connection.readyState === 1) { + // // store log + // MongoLog.create({ + // text: msg, + // level, + // metadata: obj + // }); + // } }, debug(msg: string, obj?: Record) { this.log(LogLevelEnum.debug, msg, obj); diff --git a/packages/service/common/system/log/constant.ts b/packages/service/common/system/log/constant.ts new file mode 100644 index 00000000000..45f28ac664e --- /dev/null +++ b/packages/service/common/system/log/constant.ts @@ -0,0 +1,10 @@ +export enum LogLevelEnum { + debug = 0, + info = 1, + warn = 2, + error = 3 +} + +export enum LogSignEnum { + slowOperation = 'slowOperation' +} diff --git a/packages/service/common/system/log/schema.ts b/packages/service/common/system/log/schema.ts new file mode 100644 index 00000000000..caca4d63314 --- /dev/null +++ b/packages/service/common/system/log/schema.ts @@ -0,0 +1,27 @@ +import { getMongoModel, Schema } from '../../../common/mongo'; +import { SystemLogType } from './type'; +import { LogLevelEnum } from './constant'; + +export const LogCollectionName = 'system_logs'; + +const SystemLogSchema = new Schema({ + text: { + type: String, + required: true + }, + level: { + type: String, + required: true, + enum: Object.values(LogLevelEnum) + }, + time: { + type: Date, + default: () => new Date() + }, + metadata: Object +}); + +SystemLogSchema.index({ time: 1 }, { expires: '15d' }); +SystemLogSchema.index({ level: 1 }); + +export const MongoLog = getMongoModel(LogCollectionName, SystemLogSchema); diff --git a/packages/service/common/system/log/type.d.ts b/packages/service/common/system/log/type.d.ts new file mode 100644 index 00000000000..4e51c4da36c --- /dev/null +++ b/packages/service/common/system/log/type.d.ts @@ -0,0 +1,9 @@ +import { LogLevelEnum, LogSignEnum } from './constant'; + +export type SystemLogType = { + _id: string; + text: string; + level: LogLevelEnum; + time: Date; + metadata?: Record; +}; diff --git a/packages/service/common/system/timerLock/schema.ts b/packages/service/common/system/timerLock/schema.ts index cbffb55a590..318dc056ab1 100644 --- a/packages/service/common/system/timerLock/schema.ts +++ b/packages/service/common/system/timerLock/schema.ts @@ -1,4 +1,4 @@ -import { connectionMongo, type Model } from '../../mongo'; +import { connectionMongo, getMongoModel, type Model } from '../../mongo'; import { timerIdMap } from './constants'; const { Schema, model, models } = connectionMongo; import { TimerLockSchemaType } from './type.d'; @@ -24,6 +24,4 @@ try { console.log(error); } -export const MongoTimerLock: Model = - models[collectionName] || model(collectionName, TimerLockSchema); -MongoTimerLock.syncIndexes(); +export const MongoTimerLock = getMongoModel(collectionName, TimerLockSchema); diff --git a/packages/service/core/app/schema.ts b/packages/service/core/app/schema.ts index 13bcd40622d..e83ba2202cc 100644 --- a/packages/service/core/app/schema.ts +++ b/packages/service/core/app/schema.ts @@ -1,6 +1,5 @@ import { AppTypeEnum } from '@fastgpt/global/core/app/constants'; -import { connectionMongo, type Model } from '../../common/mongo'; -const { Schema, model, models } = connectionMongo; +import { Schema, getMongoModel } from '../../common/mongo'; import type { AppSchema as AppType } from '@fastgpt/global/core/app/type.d'; import { TeamCollectionName, @@ -21,6 +20,7 @@ export const chatConfigType = { chatInputGuide: Object }; +// schema const AppSchema = new Schema({ parentId: { type: Schema.Types.ObjectId, @@ -112,15 +112,8 @@ const AppSchema = new Schema({ ...getPermissionSchema(AppDefaultPermissionVal) }); -try { - AppSchema.index({ updateTime: -1 }); - AppSchema.index({ teamId: 1, type: 1 }); - AppSchema.index({ scheduledTriggerConfig: 1, intervalNextTime: -1 }); -} catch (error) { - console.log(error); -} +AppSchema.index({ updateTime: -1 }); +AppSchema.index({ teamId: 1, type: 1 }); +AppSchema.index({ scheduledTriggerConfig: 1, intervalNextTime: -1 }); -export const MongoApp: Model = - models[AppCollectionName] || model(AppCollectionName, AppSchema); - -MongoApp.syncIndexes(); +export const MongoApp = getMongoModel(AppCollectionName, AppSchema); diff --git a/packages/service/core/app/version/schema.ts b/packages/service/core/app/version/schema.ts index 1eb5fab3ffd..e0eceb4499b 100644 --- a/packages/service/core/app/version/schema.ts +++ b/packages/service/core/app/version/schema.ts @@ -1,4 +1,4 @@ -import { connectionMongo, type Model } from '../../../common/mongo'; +import { connectionMongo, getMongoModel, type Model } from '../../../common/mongo'; const { Schema, model, models } = connectionMongo; import { AppVersionSchemaType } from '@fastgpt/global/core/app/version'; import { chatConfigType } from '../schema'; @@ -34,7 +34,7 @@ try { console.log(error); } -export const MongoAppVersion: Model = - models[AppVersionCollectionName] || model(AppVersionCollectionName, AppVersionSchema); - -MongoAppVersion.syncIndexes(); +export const MongoAppVersion = getMongoModel( + AppVersionCollectionName, + AppVersionSchema +); diff --git a/packages/service/core/chat/chatItemSchema.ts b/packages/service/core/chat/chatItemSchema.ts index 78a212c6a99..e549ef601b9 100644 --- a/packages/service/core/chat/chatItemSchema.ts +++ b/packages/service/core/chat/chatItemSchema.ts @@ -1,4 +1,4 @@ -import { connectionMongo, type Model } from '../../common/mongo'; +import { connectionMongo, getMongoModel, type Model } from '../../common/mongo'; const { Schema, model, models } = connectionMongo; import { ChatItemSchema as ChatItemType } from '@fastgpt/global/core/chat/type'; import { ChatRoleMap } from '@fastgpt/global/core/chat/constants'; @@ -9,7 +9,6 @@ import { } from '@fastgpt/global/support/user/team/constant'; import { AppCollectionName } from '../app/schema'; import { userCollectionName } from '../../support/user/schema'; -import { NodeOutputKeyEnum } from '@fastgpt/global/core/workflow/constants'; import { DispatchNodeResponseKeyEnum } from '@fastgpt/global/core/workflow/runtime/constants'; export const ChatItemCollectionName = 'chatitems'; @@ -99,7 +98,4 @@ try { console.log(error); } -export const MongoChatItem: Model = - models[ChatItemCollectionName] || model(ChatItemCollectionName, ChatItemSchema); - -MongoChatItem.syncIndexes(); +export const MongoChatItem = getMongoModel(ChatItemCollectionName, ChatItemSchema); diff --git a/packages/service/core/chat/chatSchema.ts b/packages/service/core/chat/chatSchema.ts index 86e44f8f361..294f65ed241 100644 --- a/packages/service/core/chat/chatSchema.ts +++ b/packages/service/core/chat/chatSchema.ts @@ -1,4 +1,4 @@ -import { connectionMongo, type Model } from '../../common/mongo'; +import { connectionMongo, getMongoModel, type Model } from '../../common/mongo'; const { Schema, model, models } = connectionMongo; import { ChatSchema as ChatType } from '@fastgpt/global/core/chat/type.d'; import { ChatSourceMap } from '@fastgpt/global/core/chat/constants'; @@ -98,6 +98,4 @@ try { console.log(error); } -export const MongoChat: Model = - models[chatCollectionName] || model(chatCollectionName, ChatSchema); -MongoChat.syncIndexes(); +export const MongoChat = getMongoModel(chatCollectionName, ChatSchema); diff --git a/packages/service/core/chat/inputGuide/schema.ts b/packages/service/core/chat/inputGuide/schema.ts index a8f14343e0c..0d784cea3f3 100644 --- a/packages/service/core/chat/inputGuide/schema.ts +++ b/packages/service/core/chat/inputGuide/schema.ts @@ -1,5 +1,5 @@ import { AppCollectionName } from '../../app/schema'; -import { connectionMongo, type Model } from '../../../common/mongo'; +import { connectionMongo, getMongoModel, type Model } from '../../../common/mongo'; const { Schema, model, models } = connectionMongo; import type { ChatInputGuideSchemaType } from '@fastgpt/global/core/chat/inputGuide/type.d'; @@ -23,7 +23,7 @@ try { console.log(error); } -export const MongoChatInputGuide: Model = - models[ChatInputGuideCollectionName] || model(ChatInputGuideCollectionName, ChatInputGuideSchema); - -MongoChatInputGuide.syncIndexes(); +export const MongoChatInputGuide = getMongoModel( + ChatInputGuideCollectionName, + ChatInputGuideSchema +); diff --git a/packages/service/core/dataset/collection/schema.ts b/packages/service/core/dataset/collection/schema.ts index f53b0cd99c2..2e1b35722ee 100644 --- a/packages/service/core/dataset/collection/schema.ts +++ b/packages/service/core/dataset/collection/schema.ts @@ -1,4 +1,4 @@ -import { connectionMongo, type Model } from '../../../common/mongo'; +import { connectionMongo, getMongoModel, type Model } from '../../../common/mongo'; const { Schema, model, models } = connectionMongo; import { DatasetCollectionSchemaType } from '@fastgpt/global/core/dataset/type.d'; import { TrainingTypeMap, DatasetCollectionTypeMap } from '@fastgpt/global/core/dataset/constants'; @@ -94,9 +94,6 @@ const DatasetCollectionSchema = new Schema({ } }); -export const MongoDatasetCollection: Model = - models[DatasetColCollectionName] || model(DatasetColCollectionName, DatasetCollectionSchema); - try { // auth file DatasetCollectionSchema.index({ teamId: 1, fileId: 1 }); @@ -111,8 +108,11 @@ try { // get forbid // DatasetCollectionSchema.index({ teamId: 1, datasetId: 1, forbid: 1 }); - - MongoDatasetCollection.syncIndexes({ background: true }); } catch (error) { console.log(error); } + +export const MongoDatasetCollection = getMongoModel( + DatasetColCollectionName, + DatasetCollectionSchema +); diff --git a/packages/service/core/dataset/data/schema.ts b/packages/service/core/dataset/data/schema.ts index dc43ed881d9..4d5e6aa40c1 100644 --- a/packages/service/core/dataset/data/schema.ts +++ b/packages/service/core/dataset/data/schema.ts @@ -1,4 +1,4 @@ -import { connectionMongo, type Model } from '../../../common/mongo'; +import { connectionMongo, getMongoModel, type Model } from '../../../common/mongo'; const { Schema, model, models } = connectionMongo; import { DatasetDataSchemaType } from '@fastgpt/global/core/dataset/type.d'; import { @@ -77,27 +77,23 @@ const DatasetDataSchema = new Schema({ rebuilding: Boolean }); -export const MongoDatasetData: Model = - models[DatasetDataCollectionName] || model(DatasetDataCollectionName, DatasetDataSchema); - -try { - // list collection and count data; list data; delete collection(relate data) - DatasetDataSchema.index({ - teamId: 1, - datasetId: 1, - collectionId: 1, - chunkIndex: 1, - updateTime: -1 - }); - // full text index - DatasetDataSchema.index({ teamId: 1, datasetId: 1, fullTextToken: 'text' }); - // Recall vectors after data matching - DatasetDataSchema.index({ teamId: 1, datasetId: 1, collectionId: 1, 'indexes.dataId': 1 }); - DatasetDataSchema.index({ updateTime: 1 }); - // rebuild data - DatasetDataSchema.index({ rebuilding: 1, teamId: 1, datasetId: 1 }); +// list collection and count data; list data; delete collection(relate data) +DatasetDataSchema.index({ + teamId: 1, + datasetId: 1, + collectionId: 1, + chunkIndex: 1, + updateTime: -1 +}); +// full text index +DatasetDataSchema.index({ teamId: 1, datasetId: 1, fullTextToken: 'text' }); +// Recall vectors after data matching +DatasetDataSchema.index({ teamId: 1, datasetId: 1, collectionId: 1, 'indexes.dataId': 1 }); +DatasetDataSchema.index({ updateTime: 1 }); +// rebuild data +DatasetDataSchema.index({ rebuilding: 1, teamId: 1, datasetId: 1 }); - MongoDatasetData.syncIndexes({ background: true }); -} catch (error) { - console.log(error); -} +export const MongoDatasetData = getMongoModel( + DatasetDataCollectionName, + DatasetDataSchema +); diff --git a/packages/service/core/dataset/schema.ts b/packages/service/core/dataset/schema.ts index b09739462dd..9871aaeea8a 100644 --- a/packages/service/core/dataset/schema.ts +++ b/packages/service/core/dataset/schema.ts @@ -1,4 +1,4 @@ -import { connectionMongo, type Model } from '../../common/mongo'; +import { connectionMongo, getMongoModel, type Model } from '../../common/mongo'; const { Schema, model, models } = connectionMongo; import { DatasetSchemaType } from '@fastgpt/global/core/dataset/type.d'; import { @@ -11,7 +11,6 @@ import { TeamCollectionName, TeamMemberCollectionName } from '@fastgpt/global/support/user/team/constant'; -import { PermissionTypeEnum, PermissionTypeMap } from '@fastgpt/global/support/permission/constant'; import { DatasetDefaultPermissionVal } from '@fastgpt/global/support/permission/dataset/constant'; export const DatasetCollectionName = 'datasets'; @@ -99,6 +98,4 @@ try { console.log(error); } -export const MongoDataset: Model = - models[DatasetCollectionName] || model(DatasetCollectionName, DatasetSchema); -MongoDataset.syncIndexes(); +export const MongoDataset = getMongoModel(DatasetCollectionName, DatasetSchema); diff --git a/packages/service/core/dataset/search/controller.ts b/packages/service/core/dataset/search/controller.ts index a48759724c4..1f906ac81a9 100644 --- a/packages/service/core/dataset/search/controller.ts +++ b/packages/service/core/dataset/search/controller.ts @@ -212,7 +212,7 @@ export async function searchDatasetData(props: SearchDatasetDataProps) { { $match: { $expr: { $eq: ['$_id', '$$collectionId'] }, - forbid: { $eq: false } // 直接在lookup阶段过滤 + forbid: { $eq: true } // 匹配被禁用的数据 } }, { @@ -226,7 +226,7 @@ export async function searchDatasetData(props: SearchDatasetDataProps) { }, { $match: { - collection: { $ne: [] } + collection: { $eq: [] } // 没有 forbid=true 的数据 } }, { diff --git a/packages/service/core/dataset/training/schema.ts b/packages/service/core/dataset/training/schema.ts index 1bc80da7072..d1ee257f1d0 100644 --- a/packages/service/core/dataset/training/schema.ts +++ b/packages/service/core/dataset/training/schema.ts @@ -1,5 +1,5 @@ /* 模型的知识库 */ -import { connectionMongo, type Model } from '../../../common/mongo'; +import { connectionMongo, getMongoModel, type Model } from '../../../common/mongo'; const { Schema, model, models } = connectionMongo; import { DatasetTrainingSchemaType } from '@fastgpt/global/core/dataset/type'; import { TrainingTypeMap } from '@fastgpt/global/core/dataset/constants'; @@ -103,7 +103,7 @@ try { console.log(error); } -export const MongoDatasetTraining: Model = - models[DatasetTrainingCollectionName] || model(DatasetTrainingCollectionName, TrainingDataSchema); - -MongoDatasetTraining.syncIndexes(); +export const MongoDatasetTraining = getMongoModel( + DatasetTrainingCollectionName, + TrainingDataSchema +); diff --git a/packages/service/core/workflow/dispatch/agent/extract.ts b/packages/service/core/workflow/dispatch/agent/extract.ts index 0c48625dc7b..fdfef863737 100644 --- a/packages/service/core/workflow/dispatch/agent/extract.ts +++ b/packages/service/core/workflow/dispatch/agent/extract.ts @@ -198,7 +198,7 @@ ${description ? `- ${description}` : ''} required: [] } }; - console.log(properties); + return { filterMessages, agentFunction diff --git a/packages/service/support/activity/promotion/schema.ts b/packages/service/support/activity/promotion/schema.ts index 263202d8ded..5db38606e31 100644 --- a/packages/service/support/activity/promotion/schema.ts +++ b/packages/service/support/activity/promotion/schema.ts @@ -1,4 +1,4 @@ -import { connectionMongo, type Model } from '../../../common/mongo'; +import { connectionMongo, getMongoModel, type Model } from '../../../common/mongo'; const { Schema, model, models } = connectionMongo; import { PromotionRecordSchema as PromotionRecordType } from '@fastgpt/global/support/activity/type.d'; @@ -29,6 +29,7 @@ const PromotionRecordSchema = new Schema({ } }); -export const MongoPromotionRecord: Model = - models['promotionRecord'] || model('promotionRecord', PromotionRecordSchema); -MongoPromotionRecord.syncIndexes(); +export const MongoPromotionRecord = getMongoModel( + 'promotionRecord', + PromotionRecordSchema +); diff --git a/packages/service/support/openapi/schema.ts b/packages/service/support/openapi/schema.ts index dff9af5a4f4..60757c0bf23 100644 --- a/packages/service/support/openapi/schema.ts +++ b/packages/service/support/openapi/schema.ts @@ -1,4 +1,4 @@ -import { connectionMongo, type Model } from '../../common/mongo'; +import { connectionMongo, getMongoModel, type Model } from '../../common/mongo'; const { Schema, model, models } = connectionMongo; import type { OpenApiSchema } from '@fastgpt/global/support/openapi/type'; import { @@ -64,6 +64,4 @@ try { console.log(error); } -export const MongoOpenApi: Model = - models['openapi'] || model('openapi', OpenApiSchema); -MongoOpenApi.syncIndexes(); +export const MongoOpenApi = getMongoModel('openapi', OpenApiSchema); diff --git a/packages/service/support/outLink/schema.ts b/packages/service/support/outLink/schema.ts index 63926f73d88..1e2373f15b4 100644 --- a/packages/service/support/outLink/schema.ts +++ b/packages/service/support/outLink/schema.ts @@ -1,4 +1,4 @@ -import { connectionMongo, type Model } from '../../common/mongo'; +import { connectionMongo, getMongoModel, type Model } from '../../common/mongo'; const { Schema, model, models } = connectionMongo; import { OutLinkSchema as SchemaType } from '@fastgpt/global/support/outLink/type'; import { @@ -90,7 +90,4 @@ try { console.log(error); } -export const MongoOutLink: Model = - models['outlinks'] || model('outlinks', OutLinkSchema); - -MongoOutLink.syncIndexes(); +export const MongoOutLink = getMongoModel('outlinks', OutLinkSchema); diff --git a/packages/service/support/permission/schema.ts b/packages/service/support/permission/schema.ts index 05a2db9eadd..8902fd6b868 100644 --- a/packages/service/support/permission/schema.ts +++ b/packages/service/support/permission/schema.ts @@ -2,7 +2,7 @@ import { TeamCollectionName, TeamMemberCollectionName } from '@fastgpt/global/support/user/team/constant'; -import { Model, connectionMongo } from '../../common/mongo'; +import { Model, connectionMongo, getMongoModel } from '../../common/mongo'; import type { ResourcePermissionType } from '@fastgpt/global/support/permission/type'; import { PerResourceTypeEnum } from '@fastgpt/global/support/permission/constant'; const { Schema, model, models } = connectionMongo; @@ -54,8 +54,7 @@ try { console.log(error); } -export const MongoResourcePermission: Model = - models[ResourcePermissionCollectionName] || - model(ResourcePermissionCollectionName, ResourcePermissionSchema); - -MongoResourcePermission.syncIndexes(); +export const MongoResourcePermission = getMongoModel( + ResourcePermissionCollectionName, + ResourcePermissionSchema +); diff --git a/packages/service/support/user/schema.ts b/packages/service/support/user/schema.ts index fff6f8d428a..d59c5b90f1b 100644 --- a/packages/service/support/user/schema.ts +++ b/packages/service/support/user/schema.ts @@ -1,4 +1,4 @@ -import { connectionMongo, type Model } from '../../common/mongo'; +import { connectionMongo, getMongoModel, type Model } from '../../common/mongo'; const { Schema, model, models } = connectionMongo; import { hashStr } from '@fastgpt/global/common/string/tools'; import type { UserModelSchema } from '@fastgpt/global/support/user/type'; @@ -74,6 +74,4 @@ try { console.log(error); } -export const MongoUser: Model = - models[userCollectionName] || model(userCollectionName, UserSchema); -MongoUser.syncIndexes(); +export const MongoUser = getMongoModel(userCollectionName, UserSchema); diff --git a/packages/service/support/user/team/teamMemberSchema.ts b/packages/service/support/user/team/teamMemberSchema.ts index beb0d491c6f..56c9b6efd32 100644 --- a/packages/service/support/user/team/teamMemberSchema.ts +++ b/packages/service/support/user/team/teamMemberSchema.ts @@ -1,4 +1,4 @@ -import { connectionMongo, type Model } from '../../../common/mongo'; +import { connectionMongo, getMongoModel, type Model } from '../../../common/mongo'; const { Schema, model, models } = connectionMongo; import { TeamMemberSchema as TeamMemberType } from '@fastgpt/global/support/user/team/type.d'; import { userCollectionName } from '../../user/schema'; @@ -49,5 +49,7 @@ try { console.log(error); } -export const MongoTeamMember: Model = - models[TeamMemberCollectionName] || model(TeamMemberCollectionName, TeamMemberSchema); +export const MongoTeamMember = getMongoModel( + TeamMemberCollectionName, + TeamMemberSchema +); diff --git a/packages/service/support/user/team/teamSchema.ts b/packages/service/support/user/team/teamSchema.ts index f784c1d4ef3..f6658a6c1c0 100644 --- a/packages/service/support/user/team/teamSchema.ts +++ b/packages/service/support/user/team/teamSchema.ts @@ -1,4 +1,4 @@ -import { connectionMongo, type Model } from '../../../common/mongo'; +import { connectionMongo, getMongoModel, type Model } from '../../../common/mongo'; const { Schema, model, models } = connectionMongo; import { TeamSchema as TeamType } from '@fastgpt/global/support/user/team/type.d'; import { userCollectionName } from '../../user/schema'; @@ -61,5 +61,4 @@ try { console.log(error); } -export const MongoTeam: Model = - models[TeamCollectionName] || model(TeamCollectionName, TeamSchema); +export const MongoTeam = getMongoModel(TeamCollectionName, TeamSchema); diff --git a/packages/service/support/user/team/teamTagsSchema.ts b/packages/service/support/user/team/teamTagsSchema.ts index 51566edba17..63e5ba31dce 100644 --- a/packages/service/support/user/team/teamTagsSchema.ts +++ b/packages/service/support/user/team/teamTagsSchema.ts @@ -1,4 +1,4 @@ -import { connectionMongo, type Model } from '../../../common/mongo'; +import { connectionMongo, getMongoModel, type Model } from '../../../common/mongo'; const { Schema, model, models } = connectionMongo; import { TeamTagSchema as TeamTagsSchemaType } from '@fastgpt/global/support/user/team/type.d'; import { @@ -32,5 +32,7 @@ try { console.log(error); } -export const MongoTeamTags: Model = - models[TeamTagsCollectionName] || model(TeamTagsCollectionName, TeamTagSchema); +export const MongoTeamTags = getMongoModel( + TeamTagsCollectionName, + TeamTagSchema +); diff --git a/packages/service/support/wallet/sub/schema.ts b/packages/service/support/wallet/sub/schema.ts index 42709e8024a..cdfca0d9deb 100644 --- a/packages/service/support/wallet/sub/schema.ts +++ b/packages/service/support/wallet/sub/schema.ts @@ -3,7 +3,7 @@ 1. type=standard: There will only be 1, and each team will have one 2. type=extraDatasetSize/extraPoints: Can buy multiple */ -import { connectionMongo, type Model } from '../../../common/mongo'; +import { connectionMongo, getMongoModel, type Model } from '../../../common/mongo'; const { Schema, model, models } = connectionMongo; import { TeamCollectionName } from '@fastgpt/global/support/user/team/constant'; import { @@ -93,5 +93,4 @@ try { console.log(error); } -export const MongoTeamSub: Model = - models[subCollectionName] || model(subCollectionName, SubSchema); +export const MongoTeamSub = getMongoModel(subCollectionName, SubSchema); diff --git a/packages/service/support/wallet/usage/schema.ts b/packages/service/support/wallet/usage/schema.ts index d84fde1c3a1..bceb1c7b70d 100644 --- a/packages/service/support/wallet/usage/schema.ts +++ b/packages/service/support/wallet/usage/schema.ts @@ -1,4 +1,4 @@ -import { connectionMongo, type Model } from '../../../common/mongo'; +import { connectionMongo, getMongoModel, type Model } from '../../../common/mongo'; const { Schema, model, models } = connectionMongo; import { UsageSchemaType } from '@fastgpt/global/support/wallet/usage/type'; import { UsageSourceMap } from '@fastgpt/global/support/wallet/usage/constants'; @@ -70,6 +70,4 @@ try { console.log(error); } -export const MongoUsage: Model = - models[UsageCollectionName] || model(UsageCollectionName, UsageSchema); -MongoUsage.syncIndexes(); +export const MongoUsage = getMongoModel(UsageCollectionName, UsageSchema); diff --git a/projects/app/.env.template b/projects/app/.env.template index 5123856050a..c12b615f36e 100644 --- a/projects/app/.env.template +++ b/projects/app/.env.template @@ -33,6 +33,6 @@ PRO_URL= # 首页路径 HOME_URL=/ # 日志等级: debug, info, warn, error -LOG_LEVEL=info -# Loki Log Path -# LOKI_LOG_URL= \ No newline at end of file +LOG_LEVEL=debug +STORE_LOG_LEVEL=warn +# Loki Log Path \ No newline at end of file