fix: tune network asset queue configuration

pubkeyapp · Feb 19, 2024 · b727706 · b727706
1 parent 74769d8
commit b727706
Show file tree

Hide file tree

Showing 3 changed files with 6 additions and 4 deletions.
diff --git a/libs/api/core/data-access/src/lib/queues/api-core-queues.module.ts b/libs/api/core/data-access/src/lib/queues/api-core-queues.module.ts
@@ -15,8 +15,8 @@ const logoUrl = 'https://avatars.githubusercontent.com/u/125477168?v=4'
         prefix: 'pubkey:api',
         connection: config.redisOptions,
         defaultJobOptions: {
-          removeOnFail: { age: 24 * 3600 },
-          removeOnComplete: { age: 24 * 3600 },
+          removeOnFail: { age: 24 * 3600, count: 10_000 },
+          removeOnComplete: { age: 3600, count: 1000 },
           keepLogs: 1000,
         },
       }),

diff --git a/libs/api/network-asset/data-access/src/lib/processors/api-network-asset-sync-queue.ts b/libs/api/network-asset/data-access/src/lib/processors/api-network-asset-sync-queue.ts
@@ -11,7 +11,7 @@ export interface ApiNetworkAssetIdentitySyncPayload {
   identity: Identity
 }
 
-@Processor(API_NETWORK_ASSET_SYNC, { concurrency: 8 })
+@Processor(API_NETWORK_ASSET_SYNC, { concurrency: parseInt(process.env['SYNC_NETWORK_ASSETS_CONCURRENT'] || '2') })
 export class ApiNetworkAssetSyncQueue extends WorkerHost {
   private readonly logger = new Logger(ApiNetworkAssetSyncQueue.name)
   constructor(private readonly core: ApiCoreService, private readonly sync: ApiNetworkAssetSyncService) {

diff --git a/libs/api/network-asset/data-access/src/lib/processors/api-network-asset-upsert-queue.ts b/libs/api/network-asset/data-access/src/lib/processors/api-network-asset-upsert-queue.ts
@@ -16,7 +16,9 @@ export interface ApiNetworkAssetUpsertPayload {
   asset: NetworkAssetInput
 }
 
-@Processor(API_NETWORK_ASSET_UPSERT_QUEUE, { concurrency: 4 })
+@Processor(API_NETWORK_ASSET_UPSERT_QUEUE, {
+  concurrency: parseInt(process.env['SYNC_NETWORK_ASSETS_CONCURRENT'] || '2'),
+})
 export class ApiNetworkAssetUpsertQueue extends WorkerHost {
   private readonly logger = new Logger(ApiNetworkAssetUpsertQueue.name)
   constructor(private readonly core: ApiCoreService, private readonly sync: ApiNetworkAssetSyncService) {