From b7277069d59c18f4af06af0b12e6621ed7488bef Mon Sep 17 00:00:00 2001 From: Bram Borggreve Date: Mon, 19 Feb 2024 22:37:05 +0000 Subject: [PATCH] fix: tune network asset queue configuration --- .../core/data-access/src/lib/queues/api-core-queues.module.ts | 4 ++-- .../src/lib/processors/api-network-asset-sync-queue.ts | 2 +- .../src/lib/processors/api-network-asset-upsert-queue.ts | 4 +++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/libs/api/core/data-access/src/lib/queues/api-core-queues.module.ts b/libs/api/core/data-access/src/lib/queues/api-core-queues.module.ts index 700a4197..6524f527 100644 --- a/libs/api/core/data-access/src/lib/queues/api-core-queues.module.ts +++ b/libs/api/core/data-access/src/lib/queues/api-core-queues.module.ts @@ -15,8 +15,8 @@ const logoUrl = 'https://avatars.githubusercontent.com/u/125477168?v=4' prefix: 'pubkey:api', connection: config.redisOptions, defaultJobOptions: { - removeOnFail: { age: 24 * 3600 }, - removeOnComplete: { age: 24 * 3600 }, + removeOnFail: { age: 24 * 3600, count: 10_000 }, + removeOnComplete: { age: 3600, count: 1000 }, keepLogs: 1000, }, }), diff --git a/libs/api/network-asset/data-access/src/lib/processors/api-network-asset-sync-queue.ts b/libs/api/network-asset/data-access/src/lib/processors/api-network-asset-sync-queue.ts index 98e5c30a..2ae3f455 100644 --- a/libs/api/network-asset/data-access/src/lib/processors/api-network-asset-sync-queue.ts +++ b/libs/api/network-asset/data-access/src/lib/processors/api-network-asset-sync-queue.ts @@ -11,7 +11,7 @@ export interface ApiNetworkAssetIdentitySyncPayload { identity: Identity } -@Processor(API_NETWORK_ASSET_SYNC, { concurrency: 8 }) +@Processor(API_NETWORK_ASSET_SYNC, { concurrency: parseInt(process.env['SYNC_NETWORK_ASSETS_CONCURRENT'] || '2') }) export class ApiNetworkAssetSyncQueue extends WorkerHost { private readonly logger = new Logger(ApiNetworkAssetSyncQueue.name) constructor(private readonly core: ApiCoreService, private readonly sync: ApiNetworkAssetSyncService) { diff --git a/libs/api/network-asset/data-access/src/lib/processors/api-network-asset-upsert-queue.ts b/libs/api/network-asset/data-access/src/lib/processors/api-network-asset-upsert-queue.ts index 4351a0e1..8afb85fa 100644 --- a/libs/api/network-asset/data-access/src/lib/processors/api-network-asset-upsert-queue.ts +++ b/libs/api/network-asset/data-access/src/lib/processors/api-network-asset-upsert-queue.ts @@ -16,7 +16,9 @@ export interface ApiNetworkAssetUpsertPayload { asset: NetworkAssetInput } -@Processor(API_NETWORK_ASSET_UPSERT_QUEUE, { concurrency: 4 }) +@Processor(API_NETWORK_ASSET_UPSERT_QUEUE, { + concurrency: parseInt(process.env['SYNC_NETWORK_ASSETS_CONCURRENT'] || '2'), +}) export class ApiNetworkAssetUpsertQueue extends WorkerHost { private readonly logger = new Logger(ApiNetworkAssetUpsertQueue.name) constructor(private readonly core: ApiCoreService, private readonly sync: ApiNetworkAssetSyncService) {