Merge pull request #927 from 18F/develop

[Feature] Add realtime events report -- Staging

.github/workflows/ci.yml

@@ -13,14 +13,30 @@ jobs:
       - name: Install node
         uses: actions/setup-node@v4
         with:
-          node-version: "lts/*"
+          node-version-file: ".nvmrc"
           cache: 'npm'
       - name: Install node dependencies
         run: npm ci
       - name: Lint javascript
         run: npm run lint
+  audit_dependencies:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Code checkout
+        uses: actions/checkout@v4
+      - name: Install node
+        uses: actions/setup-node@v4
+        with:
+          node-version-file: ".nvmrc"
+          cache: 'npm'
+      - name: Install node dependencies
+        run: npm ci
+      - name: Validate npm package signatures
+        run: npm audit signatures
   test:
-    needs: lint
+    needs:
+      - lint
+      - audit_dependencies
     runs-on: ubuntu-latest
     # Start Postgres as a service, wait until healthy. Uses latest Postgres version.
     services:
@@ -43,7 +59,7 @@ jobs:
       - name: Install node
         uses: actions/setup-node@v4
         with:
-          node-version: "lts/*"
+          node-version-file: ".nvmrc"
           cache: 'npm'
       - name: Install node dependencies
         run: npm ci
@@ -52,6 +68,7 @@ jobs:
   deploy_dev:
     needs:
       - lint
+      - audit_dependencies
       - test
     if: github.ref == 'refs/heads/develop'
     uses: 18F/analytics-reporter/.github/workflows/deploy.yml@develop
@@ -81,6 +98,7 @@ jobs:
   deploy_stg:
     needs:
       - lint
+      - audit_dependencies
       - test
     if: github.ref == 'refs/heads/staging'
     uses: 18F/analytics-reporter/.github/workflows/deploy.yml@develop
@@ -110,6 +128,7 @@ jobs:
   deploy_prd:
     needs:
       - lint
+      - audit_dependencies
       - test
     if: github.ref == 'refs/heads/master'
     uses: 18F/analytics-reporter/.github/workflows/deploy.yml@develop

.github/workflows/deploy.yml

@@ -91,7 +91,7 @@ jobs:
       - name: Install node
         uses: actions/setup-node@v4
         with:
-          node-version: "lts/*"
+          node-version-file: ".nvmrc"
           cache: 'npm'
       - name: Install node dependencies
         # This causes npm install to omit dev dependencies per NPM docs.
@@ -133,7 +133,7 @@ jobs:
       - name: Install node
         uses: actions/setup-node@v4
         with:
-          node-version: "lts/*"
+          node-version-file: ".nvmrc"
           cache: 'npm'
       - name: Install node dependencies
         # This causes npm install to omit dev dependencies per NPM docs.

.nvmrc

@@ -1 +1 @@
-20.11
+22

package.json

@@ -12,6 +12,8 @@
     "migrate": "knex migrate:latest",
     "pretest": "NODE_ENV=test npm run migrate",
     "start": "./bin/analytics",
+    "start:publisher": "dotenv -e .env.analytics node -- deploy/cron.js",
+    "start:consumer": "dotenv -e .env.analytics node -- deploy/consumer.js",
     "test": "NODE_ENV=test mocha",
     "test:debug": "NODE_ENV=test node --inspect-brk node_modules/mocha/bin/mocha",
     "coverage": "nyc --reporter html --reporter text -t coverage --report-dir coverage/summary npm run test",
@@ -52,7 +54,7 @@
     "*.md"
   ],
   "engines": {
-    "node": "20.x.x"
+    "node": "22.x.x"
   },
   "preferGlobal": true,
   "main": "index",
@@ -71,7 +73,6 @@
     "@google-analytics/data": "^4.7.0",
     "@smithy/node-http-handler": "^3.0.0",
     "@snyk/protect": "^1.1269.0",
-    "dotenv": "^16.3.1",
     "fast-csv": "^4.3.6",
     "googleapis": "^140.0.0",
     "max-listeners-exceeded-warning": "^0.0.1",
@@ -85,6 +86,8 @@
     "@cucumber/cucumber": "^10.3.1",
     "@eslint/js": "^8.57.0",
     "chai": "^4.4.0",
+    "dotenv": "^16.4.5",
+    "dotenv-cli": "^7.4.3",
     "eslint": "^8.56.0",
     "eslint-config-prettier": "^9.1.0",
     "eslint-plugin-jsdoc": "^48.7.0",

reports/usa.json

@@ -8,12 +8,36 @@
         "metrics": [
           {
             "name": "activeUsers"
+          },
+          {
+            "name": "screenPageViews"
+          }
+        ]
+      },
+      "meta": {
+        "name": "Active users and page views realtime",
+        "description": "Number of users currently visiting all sites and the number of page views."
+      }
+    },
+    {
+      "name": "realtime-event-counts",
+      "frequency": "realtime",
+      "realtime": true,
+      "query": {
+        "dimensions": [
+          {
+            "name": "eventName"
+          }
+        ],
+        "metrics": [
+          {
+            "name": "eventCount"
           }
         ]
       },
       "meta": {
-        "name": "Active Users Right Now",
-        "description": "Number of users currently visiting all sites."
+        "name": "Number of events of each type realtime",
+        "description": "Number of current events for each event name across all sites."
       }
     },
     {

test/actions/action.test.js

@@ -41,12 +41,10 @@ describe("Action", () => {
         }
       }
 
-      beforeEach((done) => {
+      beforeEach(async () => {
         promiseExecuted = false;
         subject = new SuccessfulTestAction();
-        subject.execute(context).then(() => {
-          done();
-        });
+        await subject.execute(context);
       });
 
       it("executes the strategy and awaits", () => {
@@ -66,13 +64,11 @@ describe("Action", () => {
         }
       }
 
-      beforeEach((done) => {
+      beforeEach(async () => {
         promiseExecuted = false;
         errorlogSpy.resetHistory();
         subject = new UnsuccessfulTestAction();
-        subject.execute(context).catch(() => {
-          done();
-        });
+        await subject.execute(context).catch(() => {});
       });
 
       it("executes the strategy and awaits", () => {

test/google_analytics/query_authorizer.test.js

@@ -39,42 +39,42 @@ describe("GoogleAnalyticsQueryAuthorizer", () => {
       appConfig.key_file = undefined;
     });
 
-    it("should resolve a query with the auth prop set to an authorized JWT", () => {
-      return subject.authorizeQuery(query, appConfig).then((query) => {
+    it("should resolve a query with the auth prop set to an authorized JWT", async () => {
+      await subject.authorizeQuery(query, appConfig).then((query) => {
         expect(query.abc).to.equal(123);
         expect(query.auth).to.not.be.undefined;
         expect(query.auth).to.be.an.instanceof(googleapis.Auth.JWT);
       });
     });
 
-    it("should create a JWT with the proper scopes", () => {
-      return subject.authorizeQuery({}, appConfig).then((query) => {
+    it("should create a JWT with the proper scopes", async () => {
+      await subject.authorizeQuery({}, appConfig).then((query) => {
         expect(query.auth.initArguments[3]).to.deep.equal([
           "https://www.googleapis.com/auth/analytics.readonly",
         ]);
       });
     });
 
-    it("should authorize the JWT and resolve if it is valid", () => {
+    it("should authorize the JWT and resolve if it is valid", async () => {
       let jwtAuthorized = false;
       googleapis.Auth.JWT.prototype.authorize = (callback) => {
         jwtAuthorized = true;
         callback(null, {});
       };
 
-      return subject.authorizeQuery({}, appConfig).then(() => {
+      await subject.authorizeQuery({}, appConfig).then(() => {
         expect(jwtAuthorized).to.equal(true);
       });
     });
 
-    it("should authorize the JWT and reject if it is invalid", () => {
+    it("should authorize the JWT and reject if it is invalid", async () => {
       let jwtAuthorized = false;
       googleapis.Auth.JWT.prototype.authorize = (callback) => {
         jwtAuthorized = true;
         callback(new Error("Failed to authorize"));
       };
 
-      return subject.authorizeQuery({}, appConfig).catch((err) => {
+      await subject.authorizeQuery({}, appConfig).catch((err) => {
         expect(jwtAuthorized).to.equal(true);
         expect(err.message).to.equal("Failed to authorize");
       });

test/process_results/result_formatter.test.js

@@ -25,10 +25,10 @@ describe("ResultFormatter", () => {
       });
     });
 
-    it("should format results into JSON if the format is 'json'", () => {
+    it("should format results into JSON if the format is 'json'", async () => {
       const result = analyticsDataProcessor.processData({ report, data });
 
-      return ResultFormatter.formatResult(result, { format: "json" }).then(
+      await ResultFormatter.formatResult(result, { format: "json" }).then(
         (formattedResult) => {
           const json = JSON.parse(formattedResult);
           json.data.forEach((row, index) => {
@@ -46,10 +46,10 @@ describe("ResultFormatter", () => {
       );
     });
 
-    it("should remove the data attribute for JSON if options.slim is true", () => {
+    it("should remove the data attribute for JSON if options.slim is true", async () => {
       const result = analyticsDataProcessor.processData({ report, data });
 
-      return ResultFormatter.formatResult(result, {
+      await ResultFormatter.formatResult(result, {
         format: "json",
         slim: true,
       }).then((formattedResult) => {
@@ -58,19 +58,19 @@ describe("ResultFormatter", () => {
       });
     });
 
-    it("should reject if the data cannot be JSON stringified", () => {
+    it("should reject if the data cannot be JSON stringified", async () => {
       const array = [];
       array[0] = array;
 
-      return ResultFormatter.formatResult(array).catch((e) => {
+      await ResultFormatter.formatResult(array).catch((e) => {
         expect(e).to.match(/TypeError: Converting circular structure to JSON/);
       });
     });
 
-    it("should format results into CSV if the format is 'csv'", () => {
+    it("should format results into CSV if the format is 'csv'", async () => {
       const result = analyticsDataProcessor.processData({ report, data });
 
-      return ResultFormatter.formatResult(result, {
+      await ResultFormatter.formatResult(result, {
         format: "csv",
         slim: true,
       }).then((formattedResult) => {
@@ -89,10 +89,10 @@ describe("ResultFormatter", () => {
       });
     });
 
-    it("should throw an error if the format is unsupported", () => {
+    it("should throw an error if the format is unsupported", async () => {
       const result = analyticsDataProcessor.processData({ report, data });
 
-      return ResultFormatter.formatResult(result, {
+      await ResultFormatter.formatResult(result, {
         format: "xml",
         slim: true,
       }).catch((e) => {