feat(GatewaysDataSource): multiple trusted gateways

docker-compose.yaml

@@ -45,6 +45,7 @@ services:
       - LOG_FORMAT=${LOG_FORMAT:-simple}
       - TRUSTED_NODE_URL=${TRUSTED_NODE_URL:-}
       - TRUSTED_GATEWAY_URL=https://${TRUSTED_GATEWAY_HOST:-arweave.net}
+      - TRUSTED_GATEWAYS_URLS=${TRUSTED_GATEWAYS_URLS:-}
       - START_HEIGHT=${START_HEIGHT:-}
       - STOP_HEIGHT=${STOP_HEIGHT:-}
       - SKIP_CACHE=${SKIP_CACHE:-}

docs/envs.md

@@ -8,6 +8,7 @@ This document describes the environment variables that can be used to configure
 | STOP_HEIGHT | Number or "Infinity" | "Infinity" | Stop block height for node synchronization (Infinity = keep syncing until stopped) |
 | TRUSTED_NODE_URL | String | "https://arweave.net" | Arweave node to use for fetching data |
 | TRUSTED_GATEWAY_URL | String | "https://arweave.net" | Arweave node to use for proxying requests |
+| TRUSTED_GATEWAYS_URLS | String | TRUSTED_GATEWAY_URL | A JSON map of gateways and priority |
 | TRUSTED_ARNS_GATEWAY_URL | String | "https://__NAME__.arweave.dev" | ArNS gateway |
 | INSTANCE_ID | String | "" | Adds an "INSTANCE_ID" field to output logs |
 | LOG_FORMAT | String | "simple" | Sets the format of output logs, accepts "simple" and "json" |
@@ -18,8 +19,8 @@ This document describes the environment variables that can be used to configure
 | ADMIN_API_KEY | String | Generated | API key used for admin API requests (if not set, it's generated and logged into the console) |
 | BACKFILL_BUNDLE_RECORDS | Boolean | false | If true, ar.io node will start indexing missing bundles |
 | FILTER_CHANGE_REPROCESS | Boolean | false | If true, all indexed bundles will be reprocessed with the new filters (you can use this when you change the filters) |
-| ON_DEMAND_RETRIEVAL_ORDER | String | s3,trusted-gateway,chunks,tx-data | Data source retrieval order for on-demand data requests |
-| BACKGROUND_RETRIEVAL_ORDER | String | chunks,s3,trusted-gateway,chunks,tx-data | Data source retrieval order for background data requests (i.e., unbundling) |
+| ON_DEMAND_RETRIEVAL_ORDER | String | s3,trusted-gateways,chunks,tx-data | Data source retrieval order for on-demand data requests |
+| BACKGROUND_RETRIEVAL_ORDER | String | chunks,s3,trusted-gateways,chunks,tx-data | Data source retrieval order for background data requests (i.e., unbundling) |
 | ANS104_UNBUNDLE_FILTER | String | {"never": true} | Only bundles compliant with this filter will be unbundled |
 | ANS104_INDEX_FILTER | String | {"never": true} | Only bundles compliant with this filter will be indexed |
 | ANS104_DOWNLOAD_WORKERS | String | 5 | Sets the number of ANS-104 bundles to attempt to download in parallel |

src/config.ts

@@ -51,6 +51,16 @@ export const TRUSTED_GATEWAY_URL = env.varOrDefault(
   'https://arweave.net',
 );
 
+// Trusted gateway URLs (for retrieving contiguous data)
+export const TRUSTED_GATEWAYS_URLS = JSON.parse(
+  env.varOrDefault(
+    'TRUSTED_GATEWAY_URLS',
+    JSON.stringify({
+      [TRUSTED_GATEWAY_URL]: 1,
+    }),
+  ),
+) as Record<string, number>;
+
 // Trusted chunk POST URLs (for posting chunks received at /chunk)
 export const CHUNK_POST_URLS = env
   .varOrDefault('CHUNK_POST_URLS', `${TRUSTED_NODE_URL}/chunk`)
@@ -82,15 +92,15 @@ export const CHUNK_POST_ABORT_TIMEOUT_MS =
 export const ON_DEMAND_RETRIEVAL_ORDER = env
   .varOrDefault(
     'ON_DEMAND_RETRIEVAL_ORDER',
-    's3,trusted-gateway,chunks,tx-data',
+    's3,trusted-gateways,chunks,tx-data',
   )
   .split(',');
 
 // Background data retrieval priority order
 export const BACKGROUND_RETRIEVAL_ORDER = env
   .varOrDefault(
     'BACKGROUND_RETRIEVAL_ORDER',
-    'chunks,s3,trusted-gateway,tx-data',
+    'chunks,s3,trusted-gateways,tx-data',
   )
   .split(',');