feat(S3DataStore): implement S3 data store

.env.bundler.example

@@ -8,8 +8,8 @@ ANS104_INDEX_FILTER={ "always": true }
 ANS104_UNBUNDLE_FILTER={ "attributes": { "owner_address": "$BUNDLER_ARWEAVE_ADDRESS" } }
 
 # Use localstack s3 bucket for shared data source between ar.io gateway and bundler
-AWS_S3_BUCKET=ar.io
-AWS_S3_PREFIX='data'
+AWS_S3_CONTIGUOUS_DATA_BUCKET=ar.io
+AWS_S3_CONTIGUOUS_DATA_PREFIX='data'
 AWS_ACCESS_KEY_ID='test'
 AWS_SECRET_ACCESS_KEY='test'
 AWS_REGION='us-east-1'

docker-compose.bundler.yaml

@@ -23,13 +23,13 @@ services:
       SQS_NEW_DATA_ITEM_URL: ${SQS_NEW_DATA_ITEM_URL:-http://sqs.us-east-1.localstack.localstack.cloud:4566/000000000000/batch-insert-new-data-items-queue}
       SQS_UNBUNDLE_BDI_URL: ${SQS_UNBUNDLE_BDI_URL:-http://sqs.us-east-1.localstack.localstack.cloud:4566/000000000000/bdi-unbundle-queue}
       OPTICAL_BRIDGING_ENABLED: ${OPTICAL_BRIDGING_ENABLED:-true}
-      DATA_ITEM_BUCKET: ${DATA_ITEM_BUCKET:-$AWS_S3_BUCKET}
+      DATA_ITEM_BUCKET: ${DATA_ITEM_BUCKET:-$AWS_S3_CONTIGUOUS_DATA_BUCKET}
       DATA_ITEM_BUCKET_REGION: ${DATA_ITEM_BUCKET_REGION:-us-east-1}
       LOG_LEVEL: ${LOG_LEVEL:-info}
       S3_FORCE_PATH_STYLE: ${S3_FORCE_PATH_STYLE:-true}
       MIGRATE_ON_STARTUP: ${MIGRATE_ON_STARTUP:-true}
-      DATA_ITEM_S3_PREFIX: ${DATA_ITEM_S3_PREFIX:-$AWS_S3_PREFIX}
-      BUNDLE_PAYLOAD_S3_PREFIX: ${BUNDLE_PAYLOAD_S3_PREFIX:-$AWS_S3_PREFIX}
+      DATA_ITEM_S3_PREFIX: ${DATA_ITEM_S3_PREFIX:-$AWS_S3_CONTIGUOUS_DATA_PREFIX}
+      BUNDLE_PAYLOAD_S3_PREFIX: ${BUNDLE_PAYLOAD_S3_PREFIX:-$AWS_S3_CONTIGUOUS_DATA_PREFIX}
     ports:
       - ${UPLOAD_SERVICE_PORT:-5100}:${UPLOAD_SERVICE_PORT:-5100}
     volumes:
@@ -66,14 +66,14 @@ services:
       OPTICAL_BRIDGING_ENABLED: ${OPTICAL_BRIDGING_ENABLED:-true}
       OPTICAL_BRIDGE_URL: ${OPTICAL_BRIDGE_URL:-http://envoy:3000/ar-io/admin/queue-data-item}
       SKIP_BALANCE_CHECKS: ${SKIP_BALANCE_CHECKS:-true}
-      DATA_ITEM_BUCKET: ${DATA_ITEM_BUCKET:-$AWS_S3_BUCKET}
+      DATA_ITEM_BUCKET: ${DATA_ITEM_BUCKET:-$AWS_S3_CONTIGUOUS_DATA_BUCKET}
       DATA_ITEM_BUCKET_REGION: ${DATA_ITEM_BUCKET_REGION:-us-east-1}
       S3_FORCE_PATH_STYLE: ${S3_FORCE_PATH_STYLE:-true}
       AWS_REGION: ${AWS_REGION:-us-east-1}
       OVERDUE_DATA_ITEM_THRESHOLD_MS: ${OVERDUE_DATA_ITEM_THRESHOLD_MS:-0} # plan all into bundles data items immediately
       AR_IO_ADMIN_KEY: ${AR_IO_ADMIN_KEY:-$ADMIN_API_KEY}
-      DATA_ITEM_S3_PREFIX: ${DATA_ITEM_S3_PREFIX:-$AWS_S3_PREFIX}
-      BUNDLE_PAYLOAD_S3_PREFIX: ${BUNDLE_PAYLOAD_S3_PREFIX:-$AWS_S3_PREFIX}
+      DATA_ITEM_S3_PREFIX: ${DATA_ITEM_S3_PREFIX:-$AWS_S3_CONTIGUOUS_DATA_PREFIX}
+      BUNDLE_PAYLOAD_S3_PREFIX: ${BUNDLE_PAYLOAD_S3_PREFIX:-$AWS_S3_CONTIGUOUS_DATA_PREFIX}
     networks:
       - ar-io-network
     depends_on:
@@ -110,7 +110,7 @@ services:
       - ARWEAVE_WALLET=${ARWEAVE_WALLET:-$BUNDLER_ARWEAVE_WALLET}
       - TURBO_OPTICAL_KEY=${TURBO_OPTICAL_KEY:-$BUNDLER_ARWEAVE_WALLET}
       - PERSIST_DEFAULT=${PERSIST_LOCALSTACK:-1} # set to 0 to disable persistence of resources and s3 objects between restarts
-      - DATA_ITEM_BUCKET=${AWS_S3_BUCKET:-}
+      - DATA_ITEM_BUCKET=${AWS_S3_CONTIGUOUS_DATA_BUCKET:-}
     volumes:
       - '${LOCALSTACK_VOLUME_DIR:-./data/localstack}:/var/lib/localstack'
       - '${LOCALSTACK_VOLUME_DIR:-./data/localstack}:/persisted-data'

docker-compose.yaml

@@ -87,8 +87,8 @@ services:
       - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-}
       - AWS_REGION=${AWS_REGION:-}
       - AWS_ENDPOINT=${AWS_ENDPOINT:-}
-      - AWS_S3_BUCKET=${AWS_S3_BUCKET:-}
-      - AWS_S3_PREFIX=${AWS_S3_PREFIX:-}
+      - AWS_S3_CONTIGUOUS_DATA_BUCKET=${AWS_S3_CONTIGUOUS_DATA_BUCKET:-}
+      - AWS_S3_CONTIGUOUS_DATA_PREFIX=${AWS_S3_CONTIGUOUS_DATA_PREFIX:-}
       - AR_IO_NODE_RELEASE=${AR_IO_NODE_RELEASE:-21-pre}
       - CHUNK_POST_URLS=${CHUNK_POST_URLS:-}
       - CHUNK_POST_RESPONSE_TIMEOUT_MS=${CHUNK_POST_RESPONSE_TIMEOUT_MS:-}

docs/envs.md

@@ -68,5 +68,5 @@ This document describes the environment variables that can be used to configure
 | AWS_SECRET_ACCESS_KEY | String | undefined | AWS secret access key for accessing AWS services |
 | AWS_REGION | String | undefined | AWS region where the resources are located |
 | AWS_ENDPOINT | String | undefined | Custom endpoint for AWS services |
-| AWS_S3_BUCKET | String | undefined | AWS S3 bucket name used for storing data |
-| AWS_S3_PREFIX | String | undefined | Prefix for the S3 bucket to organize data |
+| AWS_S3_CONTIGUOUS_DATA_BUCKET | String | undefined | AWS S3 bucket name used for storing data |
+| AWS_S3_CONTIGUOUS_DATA_PREFIX | String | undefined | Prefix for the S3 bucket to organize data |

package.json

@@ -9,8 +9,8 @@
   },
   "dependencies": {
     "@ar.io/sdk": "^2.2.5",
-    "@aws-lite/client": "^0.21.7",
-    "@aws-lite/s3": "^0.1.21",
+    "@aws-lite/client": "^0.22.4",
+    "@aws-lite/s3": "^0.2.6",
     "@clickhouse/client": "^1.3.0",
     "@permaweb/aoconnect": "^0.0.56",
     "apollo-server-express": "^3.13.0",
@@ -52,7 +52,7 @@
     "yesql": "^7.0.0"
   },
   "devDependencies": {
-    "@aws-lite/s3-types": "^0.1.24",
+    "@aws-lite/s3-types": "^0.2.6",
     "@swc/core": "^1.3.100",
     "@testcontainers/localstack": "^10.9.0",
     "@trivago/prettier-plugin-sort-imports": "^4.3.0",

src/config.ts

@@ -391,8 +391,13 @@ export const AWS_SECRET_ACCESS_KEY = env.varOrUndefined(
 );
 export const AWS_REGION = env.varOrUndefined('AWS_REGION');
 export const AWS_ENDPOINT = env.varOrUndefined('AWS_ENDPOINT');
-export const AWS_S3_BUCKET = env.varOrUndefined('AWS_S3_BUCKET');
-export const AWS_S3_PREFIX = env.varOrUndefined('AWS_S3_PREFIX');
+
+export const AWS_S3_CONTIGUOUS_DATA_BUCKET = env.varOrUndefined(
+  'AWS_S3_CONTIGUOUS_DATA_BUCKET',
+);
+export const AWS_S3_CONTIGUOUS_DATA_PREFIX = env.varOrUndefined(
+  'AWS_S3_CONTIGUOUS_DATA_PREFIX',
+);
 
 //
 // Development and testing