Allow disabling of days late limit

snowplow · Mar 16, 2023 · c99416f · c99416f
1 parent f8fd189
commit c99416f
Show file tree

Hide file tree

Showing 4 changed files with 45 additions and 16 deletions.
diff --git a/CHANGELOG b/CHANGELOG
@@ -1,3 +1,14 @@
+Snowplow Normalize 0.2.3 (2023-0X-XX)
+---------------------------------------
+## Summary
+This release allows users to disable the days late data filter to enable normalizing of events that don't populate the `dvce_sent_tstamp` field.
+
+## Features
+- Allow disabling of days late filter by setting `snowplow__days_late_allowed` to `-1`
+
+## Upgrading
+To upgrade the package, bump the version number in the packages.yml file in your project. 
+
 Snowplow Normalize 0.2.2 (2023-03-13)
 ---------------------------------------
 ## Summary

diff --git a/models/base/scratch/bigquery/snowplow_normalize_base_events_this_run.sql b/models/base/scratch/bigquery/snowplow_normalize_base_events_this_run.sql
@@ -17,14 +17,20 @@ from (
 
   from {{ var('snowplow__events') }} as a
 
-  where a.dvce_sent_tstamp <= {{ snowplow_utils.timestamp_add('day', var("snowplow__days_late_allowed", 3), 'a.dvce_created_tstamp') }}
-  and a.collector_tstamp >= {{ lower_limit }}
-  and a.collector_tstamp <= {{ upper_limit }}
-  {% if var('snowplow__derived_tstamp_partitioned', true) and target.type == 'bigquery' | as_bool() %}
-    and a.derived_tstamp >= {{ snowplow_utils.timestamp_add('hour', -1, lower_limit) }}
-    and a.derived_tstamp <= {{ upper_limit }}
-  {% endif %}
-  and {{ snowplow_utils.app_id_filter(var("snowplow__app_id",[])) }}
+  where
+    {# dvce_sent_tstamp is an optional field and not all trackers/webhooks populate it, this means this filter needs to be optional #}
+    {% if var("snowplow__days_late_allowed") == -1 %}
+      1 = 1
+    {% else %}
+      a.dvce_sent_tstamp <= {{ snowplow_utils.timestamp_add('day', var("snowplow__days_late_allowed", 3), 'a.dvce_created_tstamp') }}
+    {% endif %}
+    and a.collector_tstamp >= {{ lower_limit }}
+    and a.collector_tstamp <= {{ upper_limit }}
+    {% if var('snowplow__derived_tstamp_partitioned', true) and target.type == 'bigquery' | as_bool() %}
+      and a.derived_tstamp >= {{ snowplow_utils.timestamp_add('hour', -1, lower_limit) }}
+      and a.derived_tstamp <= {{ upper_limit }}
+    {% endif %}
+    and {{ snowplow_utils.app_id_filter(var("snowplow__app_id",[])) }}
 
 ) e
 group by e.event_id
diff --git a/models/base/scratch/databricks/snowplow_normalize_base_events_this_run.sql b/models/base/scratch/databricks/snowplow_normalize_base_events_this_run.sql
@@ -11,9 +11,15 @@ select
 
 from {{ var('snowplow__events') }} as a
 
-where a.dvce_sent_tstamp <= {{ snowplow_utils.timestamp_add('day', var("snowplow__days_late_allowed", 3), 'a.dvce_created_tstamp') }}
-and a.collector_tstamp >= {{ lower_limit }}
-and a.collector_tstamp <= {{ upper_limit }}
-and {{ snowplow_utils.app_id_filter(var("snowplow__app_id",[])) }}
+where
+  {# dvce_sent_tstamp is an optional field and not all trackers/webhooks populate it, this means this filter needs to be optional #}
+    {% if var("snowplow__days_late_allowed") == -1 %}
+      1 = 1
+    {% else %}
+      a.dvce_sent_tstamp <= {{ snowplow_utils.timestamp_add('day', var("snowplow__days_late_allowed", 3), 'a.dvce_created_tstamp') }}
+    {% endif %}
+    and a.collector_tstamp >= {{ lower_limit }}
+    and a.collector_tstamp <= {{ upper_limit }}
+    and {{ snowplow_utils.app_id_filter(var("snowplow__app_id",[])) }}
 
 qualify row_number() over (partition by a.event_id order by a.collector_tstamp, a.etl_tstamp) = 1
diff --git a/models/base/scratch/snowflake/snowplow_normalize_base_events_this_run.sql b/models/base/scratch/snowflake/snowplow_normalize_base_events_this_run.sql
@@ -13,9 +13,15 @@ select
 from {{ var('snowplow__events') }} as a
 
 
-where a.dvce_sent_tstamp <= {{ snowplow_utils.timestamp_add('day', var("snowplow__days_late_allowed", 3), 'a.dvce_created_tstamp') }}
-and a.collector_tstamp >= {{ lower_limit }}
-and a.collector_tstamp <= {{ upper_limit }}
-and {{ snowplow_utils.app_id_filter(var("snowplow__app_id",[])) }}
+where
+  {# dvce_sent_tstamp is an optional field and not all trackers/webhooks populate it, this means this filter needs to be optional #}
+    {% if var("snowplow__days_late_allowed") == -1 %}
+      1 = 1
+    {% else %}
+      a.dvce_sent_tstamp <= {{ snowplow_utils.timestamp_add('day', var("snowplow__days_late_allowed", 3), 'a.dvce_created_tstamp') }}
+    {% endif %}
+    and a.collector_tstamp >= {{ lower_limit }}
+    and a.collector_tstamp <= {{ upper_limit }}
+    and {{ snowplow_utils.app_id_filter(var("snowplow__app_id",[])) }}
 
 qualify row_number() over (partition by a.event_id order by a.collector_tstamp) = 1