feat: add Redis and BullMQ

.env.example

@@ -36,6 +36,8 @@ AUTH_REGISTER_ENABLED=true
 AUTH_SOLANA_ADMIN_IDS=
 # Enable login with Solana
 AUTH_SOLANA_ENABLED=true
+# Enable Bull UI
+BULL_ADMIN=admin:6c9107073a49d1e6129bfba07d494050c182d7630d3c86aca94ffa43cf1533f2
 # Domains to allow cookies for (comma-separated)
 COOKIE_DOMAINS=localhost,127.0.0.1
 # URL of the database to connect to
@@ -54,6 +56,8 @@ JWT_SECRET=
 HOST=127.0.0.1
 # Port to listen on
 PORT=3000
+# Redis configuration
+REDIS_URL=redis://localhost:6379
 # Session Secret (generate a random string with `openssl rand -hex 32`)
 SESSION_SECRET=
 # The URL of the Web UI, used to redirect to the Web UI after login.

docker-compose.yml

@@ -11,3 +11,9 @@ services:
       POSTGRES_PASSWORD: pubkey-link
     volumes:
       - ./tmp/postgres:/var/lib/postgresql/data
+  redis:
+    image: redis:7-alpine
+    ports:
+      - '6379:6379'
+    volumes:
+      - ./tmp/redis:/data

libs/api/core/data-access/src/lib/api-core-data-access.module.ts

@@ -1,45 +1,22 @@
-import { ApolloDriver, ApolloDriverConfig } from '@nestjs/apollo'
 import { Module } from '@nestjs/common'
-import { ConfigModule } from '@nestjs/config'
-import { GraphQLModule } from '@nestjs/graphql'
 import { ScheduleModule } from '@nestjs/schedule'
 import { ServeStaticModule } from '@nestjs/serve-static'
-import { join } from 'path'
-
-import { ApiCoreConfigService } from './api-core-config.service'
 import { ApiCoreProvisionService } from './api-core-provision.service'
 import { ApiCoreService } from './api-core.service'
-import { configuration } from './config/configuration'
-import { validationSchema } from './config/validation-schema'
-import { AppContext } from './entity/app-context'
+import { ApiCoreConfigModule } from './config/api-core-config.module'
+import { ApiCoreGraphQLModule } from './graphql/api-core-graphql.module'
 import { serveStaticFactory } from './helpers/serve-static-factory'
+import { ApiCoreQueuesModule } from './queues/api-core-queues.module'
 
 @Module({
   imports: [
-    ConfigModule.forRoot({
-      cache: true,
-      load: [configuration],
-      validationSchema,
-    }),
-    GraphQLModule.forRoot<ApolloDriverConfig>({
-      autoSchemaFile: join(process.cwd(), 'api-schema.graphql'),
-      sortSchema: true,
-      driver: ApolloDriver,
-      introspection: process.env['GRAPHQL_PLAYGROUND']?.toLowerCase() === 'true',
-      playground: {
-        settings: {
-          'request.credentials': 'include',
-        },
-      },
-      resolvers: {
-        // JSON: GraphQLJSON,
-      },
-      context: ({ req, res }: AppContext) => ({ req, res }),
-    }),
+    ApiCoreConfigModule,
+    ApiCoreGraphQLModule,
+    ApiCoreQueuesModule,
     ScheduleModule.forRoot(),
     ServeStaticModule.forRootAsync({ useFactory: serveStaticFactory() }),
   ],
-  providers: [ApiCoreService, ApiCoreConfigService, ApiCoreProvisionService],
+  providers: [ApiCoreService, ApiCoreProvisionService],
   exports: [ApiCoreService],
 })
 export class ApiCoreDataAccessModule {}

libs/api/core/data-access/src/lib/api-core.service.ts

@@ -1,8 +1,8 @@
 import { Injectable } from '@nestjs/common'
 import { CommunityRole, IdentityProvider, LogLevel, Prisma } from '@prisma/client'
 import { LogRelatedType } from '@pubkey-link/sdk'
-import { ApiCoreConfigService } from './api-core-config.service'
 import { ApiCorePrismaClient, prismaClient } from './api-core-prisma-client'
+import { ApiCoreConfigService } from './config/api-core-config.service'
 import { slugifyId } from './helpers/slugify-id'
 
 @Injectable()
@@ -78,8 +78,10 @@ export class ApiCoreService {
 }
 
 export interface CoreLogInput {
+  botId?: string | null
   data?: Prisma.InputJsonValue
   level: LogLevel
   relatedId?: string | null
   relatedType?: LogRelatedType | null
+  userId?: string | null
 }

libs/api/core/data-access/src/lib/config/api-core-config.module.ts

@@ -0,0 +1,19 @@
+import { Module } from '@nestjs/common'
+import { ConfigModule } from '@nestjs/config'
+
+import { ApiCoreConfigService } from './api-core-config.service'
+import { configuration } from './configuration'
+import { validationSchema } from './validation-schema'
+
+@Module({
+  imports: [
+    ConfigModule.forRoot({
+      cache: true,
+      load: [configuration],
+      validationSchema,
+    }),
+  ],
+  providers: [ApiCoreConfigService],
+  exports: [ApiCoreConfigService],
+})
+export class ApiCoreConfigModule {}

libs/api/core/data-access/src/lib/api-core-config.service.ts → libs/api/core/data-access/src/lib/config/api-core-config.service.ts

@@ -1,9 +1,10 @@
 import { Injectable, Logger } from '@nestjs/common'
 import { ConfigService } from '@nestjs/config'
 import { IdentityProvider } from '@prisma/client'
+import { RedisOptions } from 'bullmq'
 import { CookieOptions } from 'express-serve-static-core'
-import { ApiCoreConfig } from './config/configuration'
-import { AppConfig } from './entity/app-config.entity'
+import { AppConfig } from '../entity/app-config.entity'
+import { ApiCoreConfig } from './configuration'
 
 @Injectable()
 export class ApiCoreConfigService {
@@ -282,7 +283,33 @@ export class ApiCoreConfigService {
   }
 
   get prefix() {
-    return 'api'
+    return '/api'
+  }
+
+  get redisOptions(): RedisOptions {
+    // Parse the Redis URL to get the host, port, and password, etc.
+    const parsed = new URL(this.redisUrl)
+
+    // The URL class encodes the password if it contains special characters, so we need to decode it.
+    // https://nodejs.org/dist/latest-v18.x/docs/api/url.html#urlpassword
+    // This caused an issue because Azure Cache for Redis generates passwords that end with an equals sign.
+    const password = parsed.password ? decodeURIComponent(parsed.password) : undefined
+
+    return {
+      host: parsed.hostname,
+      port: Number(parsed.port),
+      password: password,
+      username: parsed.username,
+      tls: parsed.protocol?.startsWith('rediss')
+        ? {
+            rejectUnauthorized: false,
+          }
+        : undefined,
+    }
+  }
+
+  get redisUrl() {
+    return this.service.get('redisUrl')
   }
 
   get sessionSecret() {

libs/api/core/data-access/src/lib/config/configuration.ts

@@ -66,6 +66,7 @@ export interface ApiCoreConfig {
   host: string
   jwtSecret: string
   port: number
+  redisUrl: string
   sessionSecret: string
   webUrl: string
 }
@@ -109,6 +110,7 @@ export function configuration(): ApiCoreConfig {
     host: process.env['HOST'] as string,
     jwtSecret: process.env['JWT_SECRET'] as string,
     port: parseInt(process.env['PORT'] as string, 10) || 3000,
+    redisUrl: process.env['REDIS_URL'] as string,
     sessionSecret: process.env['SESSION_SECRET'] as string,
     webUrl: WEB_URL,
   }

libs/api/core/data-access/src/lib/config/validation-schema.ts

@@ -45,6 +45,7 @@ export const validationSchema = Joi.object({
   HOST: Joi.string().default('0.0.0.0'),
   NODE_ENV: Joi.string().valid('development', 'production', 'test', 'provision').default('development'),
   PORT: Joi.number().default(3000),
+  REDIS_URL: Joi.string().required().error(new Error(`REDIS_URL is required.`)),
   SESSION_SECRET: Joi.string().required(),
   SYNC_DRY_RUN: Joi.boolean().default(false),
 })

libs/api/core/data-access/src/lib/graphql/api-core-graphql.module.ts

@@ -0,0 +1,26 @@
+import { ApolloDriver, ApolloDriverConfig } from '@nestjs/apollo'
+import { Module } from '@nestjs/common'
+import { GraphQLModule } from '@nestjs/graphql'
+import { AppContext } from '../entity/app-context'
+import { join } from 'path'
+
+@Module({
+  imports: [
+    GraphQLModule.forRoot<ApolloDriverConfig>({
+      autoSchemaFile: join(process.cwd(), 'api-schema.graphql'),
+      sortSchema: true,
+      driver: ApolloDriver,
+      introspection: process.env['GRAPHQL_PLAYGROUND']?.toLowerCase() === 'true',
+      playground: {
+        settings: {
+          'request.credentials': 'include',
+        },
+      },
+      resolvers: {
+        // JSON: GraphQLJSON,
+      },
+      context: ({ req, res }: AppContext) => ({ req, res }),
+    }),
+  ],
+})
+export class ApiCoreGraphQLModule {}

libs/api/core/data-access/src/lib/helpers/serve-static-factory.ts

@@ -17,7 +17,7 @@ export function serveStaticFactory() {
     return [
       {
         rootPath,
-        exclude: ['/api/*', '/graphql'],
+        exclude: ['/api/*', '/graphql', '/queues/'],
       },
     ]
   }

libs/api/core/data-access/src/lib/queues/api-core-queues.module.ts

@@ -0,0 +1,37 @@
+import { ExpressAdapter } from '@bull-board/express'
+import { BullBoardModule } from '@bull-board/nestjs'
+import { BullModule } from '@nestjs/bullmq'
+import { Module } from '@nestjs/common'
+import { ApiCoreConfigModule } from '../config/api-core-config.module'
+import { ApiCoreConfigService } from '../config/api-core-config.service'
+import { BullDashboardMiddleware } from './bull-dashboard-middleware'
+
+const logoUrl = 'https://avatars.githubusercontent.com/u/125477168?v=4'
+@Module({
+  imports: [
+    BullModule.forRootAsync({
+      imports: [ApiCoreConfigModule],
+      useFactory: async (config: ApiCoreConfigService) => ({
+        prefix: 'pubkey:api',
+        connection: config.redisOptions,
+        defaultJobOptions: {
+          removeOnFail: { age: 24 * 3600 },
+        },
+      }),
+      inject: [ApiCoreConfigService],
+    }),
+    BullBoardModule.forRoot({
+      route: '/queues',
+      boardOptions: {
+        uiConfig: {
+          boardTitle: 'PubKey API',
+          boardLogo: { path: logoUrl },
+          favIcon: { default: logoUrl, alternative: logoUrl },
+        },
+      },
+      adapter: ExpressAdapter,
+      middleware: BullDashboardMiddleware,
+    }),
+  ],
+})
+export class ApiCoreQueuesModule {}

libs/api/core/data-access/src/lib/queues/bull-dashboard-middleware.ts

@@ -0,0 +1,22 @@
+import { HttpStatus, Logger, NestMiddleware } from '@nestjs/common'
+import { NextFunction, Request, Response } from 'express-serve-static-core'
+import * as process from 'process'
+
+export class BullDashboardMiddleware implements NestMiddleware {
+  private readonly envCreds = process.env['BULL_ADMIN'] ?? `admin:${Math.random().toString(36).substring(7)}`
+  private readonly encodedCreds = Buffer.from(this.envCreds).toString('base64')
+
+  constructor() {
+    Logger.verbose(`BullDashboardMiddleware: ${this.envCreds}`)
+  }
+  use(req: Request, res: Response, next: NextFunction) {
+    const reqCreds = req.get('authorization')?.split('Basic ')?.[1] ?? null
+
+    if (this.encodedCreds && reqCreds !== this.encodedCreds) {
+      res.setHeader('WWW-Authenticate', 'Basic realm="realm", charset="UTF-8"')
+      res.sendStatus(HttpStatus.UNAUTHORIZED)
+    } else {
+      next()
+    }
+  }
+}

libs/api/network/data-access/src/lib/api-network.service.ts

@@ -18,6 +18,10 @@ export class ApiNetworkService {
   private readonly cacheAnybodiesVaults = new LRUCache<string, AnybodiesVaultSnapshot>({
     max: 1000,
     ttl: 1000 * 60 * 60, // 1 hour
+    fetchMethod: async (vaultId) => {
+      this.logger.verbose(`cacheAnybodiesVaults: Cache miss for ${vaultId}`)
+      return getAnybodiesVaultSnapshot({ vaultId })
+    },
   })
 
   private readonly solanaNetworkAssetsCache = new LRUCache<string, NetworkAsset[]>({
@@ -255,7 +259,6 @@ export class ApiNetworkService {
 
       return res
     })
-    // .then((accounts) => ({ owner, accounts, amount: `${accounts.length}` }))
   }
 
   async resolveSolanaNonFungibleAssetsForOwners({
@@ -382,12 +385,8 @@ export class ApiNetworkService {
   }
 
   private async getCachedAnybodiesVaultSnapshot({ vaultId }: { vaultId: string }): Promise<AnybodiesVaultSnapshot> {
-    if (!this.cacheAnybodiesVaults.has(vaultId)) {
-      this.logger.verbose(`getCachedAnybodiesVaultSnapshot: Cache miss for vaultId: ${vaultId}`)
-      const assets = await getAnybodiesVaultSnapshot({ vaultId })
-      this.cacheAnybodiesVaults.set(vaultId, assets)
-    }
-    return this.cacheAnybodiesVaults.get(vaultId) ?? []
+    const result = await this.cacheAnybodiesVaults.fetch(vaultId)
+    return result ?? []
   }
 }
 /**

package.json

@@ -30,6 +30,9 @@
   "private": true,
   "dependencies": {
     "@apollo/server": "^4.10.0",
+    "@bull-board/api": "^5.14.0",
+    "@bull-board/express": "^5.14.0",
+    "@bull-board/nestjs": "^5.14.0",
     "@clack/prompts": "^0.7.0",
     "@coral-xyz/anchor": "^0.29.0",
     "@discordjs/rest": "^2.2.0",
@@ -45,6 +48,7 @@
     "@metaplex-foundation/umi-rpc-web3js": "^0.9.0",
     "@mrleebo/prisma-ast": "^0.8.0",
     "@nestjs/apollo": "^12.0.11",
+    "@nestjs/bullmq": "^10.1.0",
     "@nestjs/common": "10.3.0",
     "@nestjs/config": "^3.1.1",
     "@nestjs/core": "10.3.0",
@@ -75,6 +79,7 @@
     "bcrypt": "^5.1.1",
     "bs58": "^5.0.0",
     "buffer": "^6.0.3",
+    "bullmq": "^5.1.9",
     "clsx": "2.1.0",
     "cookie-parser": "^1.4.6",
     "discord.js": "^14.14.1",
@@ -143,6 +148,7 @@
     "@swc/jest": "0.2.29",
     "@testing-library/react": "14.1.2",
     "@types/bcrypt": "^5.0.2",
+    "@types/express": "^4.17.21",
     "@types/jest": "^29.5.11",
     "@types/node": "20.10.8",
     "@types/passport-discord": "^0.1.11",