Merge pull request #132 from snowplow/release/web/redshift/1.3.0

Release/web/redshift/1.3.0
snowplow · Jun 7, 2022 · d052d65 · d052d65
2 parents eaeb04a + d0ce9ec
commit d052d65
Show file tree

Hide file tree

Showing 34 changed files with 131 additions and 53 deletions.
diff --git a/.test/great_expectations/expectations/web/v1/base_redshift.json b/.test/great_expectations/expectations/web/v1/base_redshift.json
@@ -286,7 +286,7 @@
   "meta": {
     "versions": {
       "test_suite_version": "1.1.1",
-      "redshift_model_version": "1.2.0"
+      "redshift_model_version": "1.3.0"
     },
     "great_expectations.__version__": "0.12.0"
   }

diff --git a/.test/great_expectations/expectations/web/v1/metadata.json b/.test/great_expectations/expectations/web/v1/metadata.json
@@ -103,7 +103,7 @@
   "meta": {
     "versions": {
       "test_suite_version": "1.1.1",
-      "redshift_model_version": "1.2.0",
+      "redshift_model_version": "1.3.0",
       "bigquery_model_version": "1.0.3",
       "snowflake_model_version": "1.0.1"
     },

diff --git a/.test/great_expectations/expectations/web/v1/page_view_in_session_values.json b/.test/great_expectations/expectations/web/v1/page_view_in_session_values.json
@@ -27,7 +27,7 @@
   "meta": {
     "versions": {
       "test_suite_version": "1.1.1",
-      "redshift_model_version": "1.2.0",
+      "redshift_model_version": "1.3.0",
       "bigquery_model_version": "1.0.3",
       "snowflake_model_version": "1.0.1"
     },

diff --git a/.test/great_expectations/expectations/web/v1/page_views.json b/.test/great_expectations/expectations/web/v1/page_views.json
@@ -225,7 +225,7 @@
   "meta": {
     "versions": {
       "test_suite_version": "1.1.1",
-      "redshift_model_version": "1.2.0",
+      "redshift_model_version": "1.3.0",
       "bigquery_model_version": "1.0.3",
       "snowflake_model_version": "1.0.1"
     },

diff --git a/.test/great_expectations/expectations/web/v1/sessions.json b/.test/great_expectations/expectations/web/v1/sessions.json
@@ -181,7 +181,7 @@
   "meta": {
     "versions": {
       "test_suite_version": "1.1.1",
-      "redshift_model_version": "1.2.0",
+      "redshift_model_version": "1.3.0",
       "bigquery_model_version": "1.0.3",
       "snowflake_model_version": "1.0.1"
     },

diff --git a/.test/great_expectations/expectations/web/v1/users.json b/.test/great_expectations/expectations/web/v1/users.json
@@ -117,7 +117,7 @@
   "meta": {
     "versions": {
       "test_suite_version": "1.1.1",
-      "redshift_model_version": "1.2.0",
+      "redshift_model_version": "1.3.0",
       "bigquery_model_version": "1.0.3",
       "snowflake_model_version": "1.0.1"
     },

diff --git a/CHANGELOG b/CHANGELOG
@@ -1,42 +1,47 @@
+Redshift Web Version 1.3.0 (2022-06-06)
+--------------------------
+Redshift web: Change SORTKEY encoding to RAW (#129) (thanks @mark-walle!)
+Redshift Web: Fix column lengths in manifest tables (#131)
+
 Snowflake Web Version 1.0.1 (2022-02-24)
 ---------------------------------------
-Snowflake Web: Update column check stored procedure (Close #125)
-Snowflake Web: Fix varchar length for pseudonymized fields (Close #122)
-Snowflake Web: Remove start_date variable from users module (Close #123)
-Snowflake Web: Update copyright notices (Close #124)
-Snowflake Web: Fix logic in users_sessions_this_run to account for sparse data (Close #120)
-Snowflake Web: Fix varchar length for yauaa columns (Close #97)
-Snowflake Web: Fix se_label column length in events_staged (Close #109)
+Snowflake Web: Update column check stored procedure (#125)
+Snowflake Web: Fix varchar length for pseudonymized fields (#122)
+Snowflake Web: Remove start_date variable from users module (#123)
+Snowflake Web: Update copyright notices (#124)
+Snowflake Web: Fix logic in users_sessions_this_run to account for sparse data (#120)
+Snowflake Web: Fix varchar length for yauaa columns (#97)
+Snowflake Web: Fix se_label column length in events_staged (#109)
 
 Snowflake Mobile Version 1.1.0 (2021-06-16)
 ---------------------------------------
-Snowflake Mobile: Update update_manifest to throw error message (Close #104)
-Snowflake Mobile: Update commit_table procedure to throw error messages (Close #103)
-Snowflake Mobile: Fix column order in events_this_run to allow for events_staged table migration (Close #102)
+Snowflake Mobile: Update update_manifest to throw error message (#104)
+Snowflake Mobile: Update commit_table procedure to throw error messages (#103)
+Snowflake Mobile: Fix column order in events_this_run to allow for events_staged table migration (#102)
 
 BigQuery Mobile Version 1.1.0 (2021-06-15)
 ---------------------------------------
-BigQuery Mobile: Add mobile_staging_reconcilition to temp tables validation config (Close #100)
-BigQuery Mobile: Fix column order in events_this_run to allow for events_staged table migration (Close #99)
+BigQuery Mobile: Add mobile_staging_reconcilition to temp tables validation config (#100)
+BigQuery Mobile: Fix column order in events_this_run to allow for events_staged table migration (#99)
 
 Redshift Mobile Version 1.1.0 (2021-05-17)
 ---------------------------------------
-Redshift Mobile: Increase session_id character limit in manifest (Close #93)
-Redshift Mobile: Add simplified configs (Close #89)
-Redshift Mobile: Set app errors module to disabled by default (Close #88)
-Redshift Mobile: Add model_tstamp to derived tables (Close #82)
-Redshift Mobile: Move app errors columns to end of table (Close #83)
-Redshift Mobile: Remove CTE from user-aggs (Close #80)
+Redshift Mobile: Increase session_id character limit in manifest (#93)
+Redshift Mobile: Add simplified configs (#89)
+Redshift Mobile: Set app errors module to disabled by default (#88)
+Redshift Mobile: Add model_tstamp to derived tables (#82)
+Redshift Mobile: Move app errors columns to end of table (#83)
+Redshift Mobile: Remove CTE from user-aggs (#80)
 
 Snowflake Mobile Version 1.0.0 (2021-05-06)
 ---------------------------------------
-Add Snowflake mobile model v1 (Close #85)
+Add Snowflake mobile model v1 (#85)
 
 BigQuery Mobile Version 1.0.0 (2021-04-26)
 ---------------------------------------
-Update licence (Close #79)
-Fix credential loading in scripts (close #72)
-Mobile: Add BigQuery mobile model v1 (close #73)
+Update licence (#79)
+Fix credential loading in scripts (#72)
+Mobile: Add BigQuery mobile model v1 (#73)
 
 Redshift Mobile Version 1.0.0 (2021-03-25)
 ---------------------------------------
@@ -51,8 +56,8 @@ Update Architecture diagram to match snowplow/snowplow (#62)
 
 BigQuery Web Version 1.0.2 (2021-03-11)
 ---------------------------------------
-BigQuery: Handle schema evolution in core contexts (close #52)
-Update SQL Runner download link to point to Github releases (close #51)
+BigQuery: Handle schema evolution in core contexts (#52)
+Update SQL Runner download link to point to Github releases (#51)
 
 BigQuery Web Version 1.0.1 (2021-02-24)
 ---------------------------------------

diff --git a/web/v1/redshift/CHANGELOG b/web/v1/redshift/CHANGELOG
@@ -1,3 +1,8 @@
+Version 1.3.0 (2022-06-06)
+--------------------------
+Redshift web: Change SORTKEY encoding to RAW (#129) (thanks @mark-walle!)
+Redshift Web: Fix column lengths in manifest tables (#131)
+
 Version 1.2.0 (2021-01-25)
 ---------------------------
 Redshift: Add config (#12)

diff --git a/web/v1/redshift/README.md b/web/v1/redshift/README.md
@@ -167,7 +167,7 @@ Detail on configuring the users module's playbook can be found [in the relevant
 
 ## A note on duplicates
 
-This version of the model (1.2.0) excludes duplicated event_ids and page_view_ids. Ideally in the future it will provide standard options for handling them, however because customisations of this model involve queries on federated atomic tables, the safest option for a first version is to exclude them.
+This version of the model (>1.1.0) excludes duplicated event_ids and page_view_ids. Ideally in the future it will provide standard options for handling them, however because customisations of this model involve queries on federated atomic tables, the safest option for a first version is to exclude them.
 
 If there is a need to handle duplicates, this can be done by adding a custom module to the base level of aggregation - take good care to manage the possibility of introducing duplicates downstream if doing so.
 

diff --git a/web/v1/redshift/sql-runner/configs/migrations.json b/web/v1/redshift/sql-runner/configs/migrations.json
@@ -0,0 +1,17 @@
+{
+    "schema": "iglu:com.snowplowanalytics.datamodeling/config/jsonschema/1-0-0",
+    "data": {
+        "enabled": true,
+        "storage": "Default",
+        "playbooks": [
+        {
+            "playbook": "standard/99-migrations/1.3.0-migration",
+            "dependsOn": []
+        }
+        ],
+        "lockType": "hard",
+        "owners": [
+        ]
+    }
+}
+
diff --git a/web/v1/redshift/sql-runner/playbooks/standard/00-setup/00-setup-metadata.yml.tmpl b/web/v1/redshift/sql-runner/playbooks/standard/00-setup/00-setup-metadata.yml.tmpl
@@ -8,7 +8,7 @@
   :password:
   :ssl:
 :variables:
-  :model_version:      redshift/web/1.2.0
+  :model_version:      redshift/web/1.3.0
   :scratch_schema:     scratch
   :output_schema:      derived
   :entropy:            ""

diff --git a/web/v1/redshift/sql-runner/playbooks/standard/00-setup/99-metadata-complete.yml.tmpl b/web/v1/redshift/sql-runner/playbooks/standard/00-setup/99-metadata-complete.yml.tmpl
@@ -8,7 +8,7 @@
   :password:
   :ssl:
 :variables:
-  :model_version:      redshift/web/1.2.0
+  :model_version:      redshift/web/1.3.0
   :scratch_schema:     scratch
   :output_schema:      derived
   :entropy:            ""

diff --git a/web/v1/redshift/sql-runner/playbooks/standard/00-setup/XX-destroy-metadata.yml.tmpl b/web/v1/redshift/sql-runner/playbooks/standard/00-setup/XX-destroy-metadata.yml.tmpl
@@ -8,7 +8,7 @@
   :password:
   :ssl:
 :variables:
-  :model_version:      redshift/web/1.2.0
+  :model_version:      redshift/web/1.3.0
   :scratch_schema:     scratch
   :output_schema:      derived
   :entropy:            ""

diff --git a/web/v1/redshift/sql-runner/playbooks/standard/01-base/01-base-main.yml.tmpl b/web/v1/redshift/sql-runner/playbooks/standard/01-base/01-base-main.yml.tmpl
@@ -8,7 +8,7 @@
   :password:
   :ssl:
 :variables:
-  :model_version:      redshift/web/1.2.0
+  :model_version:      redshift/web/1.3.0
   :input_schema:       atomic
   :scratch_schema:     scratch
   :output_schema:      derived

diff --git a/web/v1/redshift/sql-runner/playbooks/standard/01-base/99-base-complete.yml.tmpl b/web/v1/redshift/sql-runner/playbooks/standard/01-base/99-base-complete.yml.tmpl
@@ -8,7 +8,7 @@
   :password:
   :ssl:
 :variables:
-  :model_version:      redshift/web/1.2.0
+  :model_version:      redshift/web/1.3.0
   :scratch_schema:     scratch
   :output_schema:      derived
   :entropy:            ""

diff --git a/web/v1/redshift/sql-runner/playbooks/standard/01-base/XX-destroy-base.yml.tmpl b/web/v1/redshift/sql-runner/playbooks/standard/01-base/XX-destroy-base.yml.tmpl
@@ -8,7 +8,7 @@
   :password:
   :ssl:
 :variables:
-  :model_version:      redshift/web/1.2.0
+  :model_version:      redshift/web/1.3.0
   :scratch_schema:     scratch
   :output_schema:      derived
   :entropy:            ""

diff --git a/web/v1/redshift/sql-runner/playbooks/standard/02-page-views/01-page-views-main.yml.tmpl b/web/v1/redshift/sql-runner/playbooks/standard/02-page-views/01-page-views-main.yml.tmpl
@@ -8,7 +8,7 @@
   :password:
   :ssl:
 :variables:
-  :model_version:      redshift/web/1.2.0
+  :model_version:      redshift/web/1.3.0
   :input_schema:       atomic
   :scratch_schema:     scratch
   :output_schema:      derived

diff --git a/web/v1/redshift/sql-runner/playbooks/standard/02-page-views/99-page-views-complete.yml.tmpl b/web/v1/redshift/sql-runner/playbooks/standard/02-page-views/99-page-views-complete.yml.tmpl
@@ -8,7 +8,7 @@
   :password:
   :ssl:
 :variables:
-  :model_version:      redshift/web/1.2.0
+  :model_version:      redshift/web/1.3.0
   :scratch_schema:     scratch
   :output_schema:      derived
   :entropy:            ""

diff --git a/web/v1/redshift/sql-runner/playbooks/standard/02-page-views/XX-destroy-page-views.yml.tmpl b/web/v1/redshift/sql-runner/playbooks/standard/02-page-views/XX-destroy-page-views.yml.tmpl
@@ -8,7 +8,7 @@
   :password:
   :ssl:
 :variables:
-  :model_version:      redshift/web/1.2.0
+  :model_version:      redshift/web/1.3.0
   :scratch_schema:     scratch
   :output_schema:      derived
   :entropy:            ""

diff --git a/web/v1/redshift/sql-runner/playbooks/standard/03-sessions/01-sessions-main.yml.tmpl b/web/v1/redshift/sql-runner/playbooks/standard/03-sessions/01-sessions-main.yml.tmpl
@@ -8,7 +8,7 @@
   :password:
   :ssl:
 :variables:
-  :model_version:      redshift/web/1.2.0
+  :model_version:      redshift/web/1.3.0
   :scratch_schema:     scratch
   :output_schema:      derived
   :entropy:            ""

diff --git a/web/v1/redshift/sql-runner/playbooks/standard/03-sessions/99-sessions-complete.yml.tmpl b/web/v1/redshift/sql-runner/playbooks/standard/03-sessions/99-sessions-complete.yml.tmpl
@@ -8,7 +8,7 @@
   :password:
   :ssl:
 :variables:
-  :model_version:      redshift/web/1.2.0
+  :model_version:      redshift/web/1.3.0
   :scratch_schema:     scratch
   :output_schema:      derived
   :entropy:            ""

diff --git a/web/v1/redshift/sql-runner/playbooks/standard/03-sessions/XX-destroy-sessions.yml.tmpl b/web/v1/redshift/sql-runner/playbooks/standard/03-sessions/XX-destroy-sessions.yml.tmpl
@@ -8,7 +8,7 @@
   :password:
   :ssl:
 :variables:
-  :model_version:      redshift/web/1.2.0
+  :model_version:      redshift/web/1.3.0
   :scratch_schema:     scratch
   :output_schema:      derived
   :entropy:            ""

diff --git a/web/v1/redshift/sql-runner/playbooks/standard/04-users/01-users-main.yml.tmpl b/web/v1/redshift/sql-runner/playbooks/standard/04-users/01-users-main.yml.tmpl
@@ -8,7 +8,7 @@
   :password:
   :ssl:
 :variables:
-  :model_version:      redshift/web/1.2.0
+  :model_version:      redshift/web/1.3.0
   :scratch_schema:     scratch
   :output_schema:      derived
   :entropy:            ""

diff --git a/web/v1/redshift/sql-runner/playbooks/standard/04-users/99-users-complete.yml.tmpl b/web/v1/redshift/sql-runner/playbooks/standard/04-users/99-users-complete.yml.tmpl
@@ -8,7 +8,7 @@
   :password:
   :ssl:
 :variables:
-  :model_version:      redshift/web/1.2.0
+  :model_version:      redshift/web/1.3.0
   :scratch_schema:     scratch
   :output_schema:      derived
   :entropy:            ""

diff --git a/web/v1/redshift/sql-runner/playbooks/standard/04-users/XX-destroy-users.yml.tmpl b/web/v1/redshift/sql-runner/playbooks/standard/04-users/XX-destroy-users.yml.tmpl
@@ -8,7 +8,7 @@
   :password:
   :ssl:
 :variables:
-  :model_version:      redshift/web/1.2.0
+  :model_version:      redshift/web/1.3.0
   :scratch_schema:     scratch
   :output_schema:      derived
   :entropy:            ""

diff --git a/web/v1/redshift/sql-runner/playbooks/standard/99-migrations/1.3.0-migration.yml.tmpl b/web/v1/redshift/sql-runner/playbooks/standard/99-migrations/1.3.0-migration.yml.tmpl
@@ -0,0 +1,34 @@
+:targets:
+- :name:
+  :type:     redshift
+  :host:
+  :database:
+  :port:
+  :username:
+  :password:
+  :ssl:
+:variables:
+  :scratch_schema:     scratch
+  :output_schema:      derived
+  :entropy:            ""
+:steps:
+- :name: 1.3.0-migration
+  :queries:
+    - :name: base-session-id-manifest
+      :file: standard/99-migrations/1.3.0-migration/base-session-id-manifest.sql
+      :template: true
+- :name: 1.3.0-migration
+  :queries:
+    - :name: sessions-userid-manifest
+      :file: standard/99-migrations/1.3.0-migration/sessions-userid-manifest.sql
+      :template: true
+- :name: 1.3.0-migration
+  :queries:
+    - :name: users-manifest
+      :file: standard/99-migrations/1.3.0-migration/users-manifest.sql
+      :template: true
+- :name: 1.3.0-migration
+  :queries:
+    - :name: sortkey-encoding
+      :file: standard/99-migrations/1.3.0-migration/sortkey-encoding.sql
+      :template: true
diff --git a/web/v1/redshift/sql-runner/sql/standard/01-base/01-main/00-setup-base.sql b/web/v1/redshift/sql-runner/sql/standard/01-base/01-main/00-setup-base.sql
@@ -108,7 +108,7 @@ INSERT INTO {{.output_schema}}.base_event_id_manifest{{.entropy}} (
 );
 
 CREATE TABLE IF NOT EXISTS {{.output_schema}}.base_session_id_manifest{{.entropy}} (
-  session_id VARCHAR(36),
+  session_id VARCHAR(128),
   min_tstamp TIMESTAMP
 )
 DISTSTYLE KEY
@@ -117,7 +117,7 @@ SORTKEY (min_tstamp);
 
 INSERT INTO {{.output_schema}}.base_session_id_manifest{{.entropy}} (
   SELECT
-    'seed'::VARCHAR(36),
+    'seed'::VARCHAR(128),
     '{{.start_date}}'::TIMESTAMP
 
   WHERE

diff --git a/web/v1/redshift/sql-runner/sql/standard/02-page-views/01-main/00-setup-page-views.sql b/web/v1/redshift/sql-runner/sql/standard/02-page-views/01-main/00-setup-page-views.sql
@@ -106,7 +106,7 @@ CREATE TABLE IF NOT EXISTS {{.output_schema}}.page_views{{.entropy}} (
   dvce_created_tstamp TIMESTAMP ENCODE ZSTD,
   collector_tstamp TIMESTAMP ENCODE ZSTD,
   derived_tstamp TIMESTAMP ENCODE ZSTD,
-  start_tstamp TIMESTAMP ENCODE ZSTD,
+  start_tstamp TIMESTAMP ENCODE RAW,
   end_tstamp TIMESTAMP ENCODE ZSTD,
 
   engaged_time_in_s INT ENCODE ZSTD,

diff --git a/web/v1/redshift/sql-runner/sql/standard/03-sessions/01-main/00-setup-sessions.sql b/web/v1/redshift/sql-runner/sql/standard/03-sessions/01-main/00-setup-sessions.sql
@@ -95,7 +95,7 @@ CREATE TABLE IF NOT EXISTS {{.output_schema}}.sessions{{.entropy}} (
   domain_sessionid VARCHAR(128) ENCODE ZSTD,
   domain_sessionidx INT ENCODE ZSTD,
 
-  start_tstamp TIMESTAMP ENCODE ZSTD,
+  start_tstamp TIMESTAMP ENCODE RAW,
   end_tstamp TIMESTAMP ENCODE ZSTD,
 
   -- user fields
@@ -220,7 +220,7 @@ SORTKEY (start_tstamp);
 
 -- Staged manifest table as input to users step
 CREATE TABLE IF NOT EXISTS {{.scratch_schema}}.sessions_userid_manifest_staged{{.entropy}} (
-  domain_userid VARCHAR(36),
+  domain_userid VARCHAR(128),
   start_tstamp TIMESTAMP
 )
 DISTSTYLE KEY