diff --git a/README.md b/README.md index 029ba08..06edab2 100644 --- a/README.md +++ b/README.md @@ -196,6 +196,23 @@ This macro only seeks to add or update the tags which are specified in dbt. It w If you need this behaviour, it usually comes naturally as dbt drops and recreates tables/views for most materializations. If you are using the incremental materialization, be aware of this limitation. +### snowflake_utils.create_udfs ([source](macros/create_udfs.sql)) +This macro will create the udfs listed in the macro (found in [udfs](macros/udfs) folder) into the target schema on-run-start. Once these UDFs are created, this allows the user to call UDFs within their SQL queries/models as part of your dbt project. + +#### Arguments +None: The [on-run-end context object](https://docs.getdbt.com/reference/dbt-jinja-functions/on-run-end-context). + +#### Usage + +The macro must be called as part of on-run-start, so add the following to dbt_project.yml: +``` +on-run-end: "{{ create_udfs() }}" +``` +#### UDFs currently generated +* udf_business_days +* udf_business_hours + +See individual yaml files within [udfs](macros/udfs/) folder to learn more about syntax and use cases. ---- diff --git a/macros/create_udfs.sql b/macros/create_udfs.sql new file mode 100644 index 0000000..104ba42 --- /dev/null +++ b/macros/create_udfs.sql @@ -0,0 +1,8 @@ +{% macro create_udfs() %} + +create schema if not exists {{target.schema}}; + +{{ create_udf_business_days() }}; +{{ create_udf_business_hours() }}; + +{% endmacro %} diff --git a/macros/macros.md b/macros/macros.md index a44a4e7..c533e77 100644 --- a/macros/macros.md +++ b/macros/macros.md @@ -123,4 +123,32 @@ This macro only seeks to add or update the tags which are specified in dbt. It w If you need this behaviour, it usually comes naturally as dbt drops and recreates tables/views for most materializations. If you are using the incremental materialization, be aware of this limitation. -{% enddocs %} \ No newline at end of file +{% enddocs %} + +{% docs create_udfs %} + +Create_udfs is a macro that does exactly as it says, calls the individiual +UDFs that are nested in macros/udfs and makes them available to use in +your dbt transformations within Snowflake. + +In order to facilitate the use of UDFs within your project, you must add +the following to your dbt_project.yml file; + +on-run-start: + - '{{ create_udfs() }}' + +Users must have the proper permissions to create UDFs, orchestrated +through the following command as an owner/administrator: +GRANT USAGE ON LANGUAGE PLPYTHONU TO + +Why/motivation: +While many complex transformations are possible with just SQL, creating +UDFs allows a project to begin encorporating python packages +to make certain transformations incredibly easy and efficient. +Additionally, leveraging UDFs via dbt macros allows for: +- the ability to version control, +- the ability to read transformations from your project without needing + to interact with Snowflake +- to maintain separate dev/prod versions of the UDFs. + +{% enddocs %} diff --git a/macros/macros.yml b/macros/macros.yml index 9434d8e..5d050d2 100644 --- a/macros/macros.yml +++ b/macros/macros.yml @@ -28,4 +28,7 @@ macros: description: '{{ doc("apply_meta_as_tags") }}' arguments: - name: results - description: The on-run-end context object \ No newline at end of file + description: The on-run-end context object + + - name: create_udfs + description: '{{ doc("create_udfs") }}' diff --git a/macros/udfs/business_days.sql b/macros/udfs/business_days.sql new file mode 100644 index 0000000..672026d --- /dev/null +++ b/macros/udfs/business_days.sql @@ -0,0 +1,91 @@ +{% macro create_udf_business_days() %} + +create or replace function {{target_schema}}.udf_business_days(start_datetime DATE, end_datetime DATE, weekmask STRING, country STRING) +returns int +language python +runtime_version = 3.8 +packages = ('numpy','holidays') +handler = 'business_days_py' +as + +$$ +import numpy as np +import holidays as holidays + +def business_days_py(start_date, end_date, weekmask, country): + # calculate business days between two dates + years = [*range(1990,2030)] + + holiday_list = list(holidays.country_holidays(country, years=years)) + + return np.busday_count( + start_date, + end_date, + weekmask = weekmask, + holidays = holiday_list + ) + +$$ + +-- overload file to default weekmask as Monday to Friday (weekmask='1111100') +create or replace function {{target_schema}}.udf_business_days(start_datetime DATE, end_datetime DATE, country STRING) +returns int +language python +runtime_version = 3.8 +packages = ('numpy','holidays') +handler = 'business_days_py' +as + +$$ +import numpy as np +import holidays as holidays + +def business_days_py(start_date, end_date, country): + + # calculate business days between two dates + years = [*range(1990,2030)] + weekmask='1111100' + + holiday_list = list(holidays.country_holidays(country, years=years)) + + return np.busday_count( + start_date, + end_date, + weekmask = weekmask, + holidays = holiday_list + ) + +$$ + +-- finally, overload function to default weekmask as M-F and country as 'US' +create or replace function {{target_schema}}.udf_business_days(start_datetime DATE, end_datetime DATE) +returns int +language python +runtime_version = 3.8 +packages = ('numpy','holidays') +handler = 'business_days_py' +as + +$$ +import numpy as np +import holidays as holidays + +def business_days_py(start_date, end_date): + + # calculate business days between two dates + years = [*range(1990,2030)] + weekmask='1111100' + country='US' + + holiday_list = list(holidays.country_holidays(country, years=years)) + + return np.busday_count( + start_date, + end_date, + weekmask = weekmask, + holidays = holiday_list + ) + +$$ + +{% endmacro %} diff --git a/macros/udfs/business_days.yml b/macros/udfs/business_days.yml new file mode 100644 index 0000000..637dcb6 --- /dev/null +++ b/macros/udfs/business_days.yml @@ -0,0 +1,49 @@ +version: 2 + +macros: + - name: create_udf_business_days + description: > + This macro creates a UDF within the target schema that can be used to + calculate business days (excluding holidays) between two date fields + in a Snowflake table. The macro mearly creates the UDF within Snowflake, + and is called by the create_udfs macro. You must ensure that create_udfs + is called on-run-start in dbt_project.yml. + + To utilize the UDF within your dbt project, proper syntax is: + + select + *, + udf_business_days(start_date, end_date, country, weekmask) + as business_days_elapsed + from sample_table + + Notes: + - The date fields passed must be of type date or datetime (timestamp_ntz). + - The country field is optional as it is defaulted to 'US'. This can be a + column within your table if you have businesses in different geographic + locations, or can be set as a string for the entire dataset. If looking + to use a different country, must use the ISO 3166-1 alpha-2 country code + as defined here: + https://en.wikipedia.org/wiki/List_of_ISO_3166_country_codes + - The weekmask field is a string indicating which days of the week are to + be considered business days, and defaults to the standard workweek being + Monday through Friday via weekmask = '1111100'; the weekmask starting + with Monday and ending with Sunday, with a 1 indicating viable business + day and 0 indicating a non-business day. This field is optional. + - Regarding defaulted arguments,calls can be made with the following + syntax: + udf_business_days(start_date, end_date, country, weekmask) + udf_business_days(start_date, end_date, country) + udf_business_days(start_date, end_date) + - This UDF only considers holidays between Jan 1 1990 and Dec 31 2030, as + is currently hardcoded. This can be changed in the macro source code. + - If increased granularity is desired for the region, you must modify the + macro to apply regional (state or province) holidays as described as + defined here: + holiday_list = holidays.country_holidays(country, subdiv='PR') + - The holidays python package does not include the end date in its + calculations of the duration. To account for this, you can add a day to + your end date to make the calculation inclusive. + + docs: + show: false diff --git a/macros/udfs/business_hours.sql b/macros/udfs/business_hours.sql new file mode 100644 index 0000000..fb100e1 --- /dev/null +++ b/macros/udfs/business_hours.sql @@ -0,0 +1,239 @@ +{% macro create_udf_business_hours() %} + +create or replace function {{target.schema}}.udf_business_hours(start_datetime TIMESTAMP_NTZ, end_datetime TIMESTAMP_NTZ, country STRING, weekmask STRING) +returns number +language python +runtime_version = 3.8 +packages = ('numpy','holidays') +handler = 'business_hours_py' +as + +$$ +from datetime import datetime +from datetime import timedelta +import holidays +import numpy as np + +def business_hours_py(start_datetime, end_datetime, country, weekmask): + + # Year range for holidays spanning the years below + + years = [*range(1990,2030)] + + # Hard coded business hours, opening and closing times + + opening_hour = 9 + closing_hour = 17 + workhours_per_day = closing_hour-opening_hour + omitted_dow = [] + + # Create list of omitted days of week for exclusion of hourly calcs from first and last days of date range + + for count, i in enumerate(weekmask): + if i=='0': + omitted_dow.append(count) + + # Create open and closing datetimes to establish day 0 and day n durations + + first_day_closed = start_datetime.replace(hour=closing_hour, minute=0, second=0) + last_day_opened = end_datetime.replace(hour=opening_hour, minute=0, second=0) + + holiday_list = list(holidays.country_holidays(country, years=years)) + days = np.busday_count( + start_datetime.date(), + end_datetime.date(), + weekmask = weekmask, + holidays = holiday_list) + + # Calculate hours for full business days (excluding first day, last day is automatically excluded in busday_count) + + complete_days_hours = (days - 1) * workhours_per_day + + # Jump forward to the next non-holiday + # When start_datetime is iterated beyond end_datetime, this indicates entirety of hours occuring during holidays or + # weekend, therefore return 0 via max(0,business_hours). + + while start_datetime.date() in holiday_list or start_datetime.weekday() in omitted_dow: + start_datetime = start_datetime.replace(hour=opening_hour,minute=0,second=0) + timedelta(days=1) + + if start_datetime.date() == end_datetime.date(): + business_hours = round((end_datetime - start_datetime).seconds/60/60,2) + return max(0, business_hours) + else: + duration_day_zero = round((first_day_closed - start_datetime ).seconds/60/60, 2) + + if end_datetime.date() in holiday_list or end_datetime.weekday() in omitted_dow: + # If end_datetime falls on a holiday or weekend, the previous full day is accounted for in + # complete_day_hours calc so early return just duration_day_zero and complete_day_hours + business_hours = round(duration_day_zero + complete_days_hours, 2) + return max(0,business_hours) + else: + # If end_datetime does not fall on holiday or weekend, calculate last days working hours + # and return total sum + duration_day_n = (end_datetime - last_day_opened).seconds/60/60 + business_hours = round(duration_day_zero + complete_days_hours + duration_day_n, 2) + return max(0, business_hours) + +$$; + +create or replace function {{target.schema}}.udf_business_hours(start_datetime TIMESTAMP_NTZ, end_datetime TIMESTAMP_NTZ, country STRING) +returns number +language python +runtime_version = 3.8 +packages = ('numpy','holidays') +handler = 'business_hours_py' +as + +$$ +from datetime import datetime +from datetime import timedelta +import holidays +import numpy as np + +def business_hours_py(start_datetime, end_datetime, country, weekmask): + + # Default values, weekmask Monday to Friday + + weekmask = '1111100' + + # Year range for holidays spanning the years below + + years = [*range(1990,2030)] + + # Hard coded business hours, opening and closing times + + opening_hour = 9 + closing_hour = 17 + workhours_per_day = closing_hour-opening_hour + omitted_dow = [] + + # Create list of omitted days of week for exclusion of hourly calcs from first and last days of date range + + for count, i in enumerate(weekmask): + if i=='0': + omitted_dow.append(count) + + # Create open and closing datetimes to establish day 0 and day n durations + + first_day_closed = start_datetime.replace(hour=closing_hour, minute=0, second=0) + last_day_opened = end_datetime.replace(hour=opening_hour, minute=0, second=0) + + holiday_list = list(holidays.country_holidays(country, years=years)) + days = np.busday_count( + start_datetime.date(), + end_datetime.date(), + weekmask = weekmask, + holidays = holiday_list) + + # Calculate hours for full business days (excluding first day, last day is automatically excluded in busday_count) + + complete_days_hours = (days - 1) * workhours_per_day + + # Jump forward to the next non-holiday + # When start_datetime is iterated beyond end_datetime, this indicates entirety of hours occuring during holidays or + # weekend, therefore return 0 via max(0,business_hours). + + while start_datetime.date() in holiday_list or start_datetime.weekday() in omitted_dow: + start_datetime = start_datetime.replace(hour=opening_hour,minute=0,second=0) + timedelta(days=1) + + if start_datetime.date() == end_datetime.date(): + business_hours = round((end_datetime - start_datetime).seconds/60/60,2) + return max(0, business_hours) + else: + duration_day_zero = round((first_day_closed - start_datetime ).seconds/60/60, 2) + + if end_datetime.date() in holiday_list or end_datetime.weekday() in omitted_dow: + # If end_datetime falls on a holiday or weekend, the previous full day is accounted for in + # complete_day_hours calc so early return just duration_day_zero and complete_day_hours + business_hours = round(duration_day_zero + complete_days_hours, 2) + return max(0,business_hours) + else: + # If end_datetime does not fall on holiday or weekend, calculate last days working hours + # and return total sum + duration_day_n = (end_datetime - last_day_opened).seconds/60/60 + business_hours = round(duration_day_zero + complete_days_hours + duration_day_n, 2) + return max(0, business_hours) + +$$; + +create or replace function {{target.schema}}.udf_business_hours(start_datetime TIMESTAMP_NTZ, end_datetime TIMESTAMP_NTZ) +returns number +language python +runtime_version = 3.8 +packages = ('numpy','holidays') +handler = 'business_hours_py' +as + +$$ +from datetime import datetime +from datetime import timedelta +import holidays +import numpy as np + +def business_hours_py(start_datetime, end_datetime): + + # Default values for country (America) and weekmask (M-F) + country = 'US' + weekmask = '1111100' + + # Year range for holidays spanning the years below + + years = [*range(1990,2030)] + + # Hard coded business hours, opening and closing times + + opening_hour = 9 + closing_hour = 17 + workhours_per_day = closing_hour-opening_hour + omitted_dow = [] + + # Create list of omitted days of week for exclusion of hourly calcs from first and last days of date range + + for count, i in enumerate(weekmask): + if i=='0': + omitted_dow.append(count) + + # Create open and closing datetimes to establish day 0 and day n durations + + first_day_closed = start_datetime.replace(hour=closing_hour, minute=0, second=0) + last_day_opened = end_datetime.replace(hour=opening_hour, minute=0, second=0) + + holiday_list = list(holidays.country_holidays(country, years=years)) + days = np.busday_count( + start_datetime.date(), + end_datetime.date(), + weekmask = weekmask, + holidays = holiday_list) + + # Calculate hours for full business days (excluding first day, last day is automatically excluded in busday_count) + + complete_days_hours = (days - 1) * workhours_per_day + + # Jump forward to the next non-holiday + # When start_datetime is iterated beyond end_datetime, this indicates entirety of hours occuring during holidays or + # weekend, therefore return 0 via max(0,business_hours). + + while start_datetime.date() in holiday_list or start_datetime.weekday() in omitted_dow: + start_datetime = start_datetime.replace(hour=opening_hour,minute=0,second=0) + timedelta(days=1) + + if start_datetime.date() == end_datetime.date(): + business_hours = round((end_datetime - start_datetime).seconds/60/60,2) + return max(0, business_hours) + else: + duration_day_zero = round((first_day_closed - start_datetime ).seconds/60/60, 2) + + if end_datetime.date() in holiday_list or end_datetime.weekday() in omitted_dow: + # If end_datetime falls on a holiday or weekend, the previous full day is accounted for in + # complete_day_hours calc so early return just duration_day_zero and complete_day_hours + business_hours = round(duration_day_zero + complete_days_hours, 2) + return max(0,business_hours) + else: + # If end_datetime does not fall on holiday or weekend, calculate last days working hours + # and return total sum + duration_day_n = (end_datetime - last_day_opened).seconds/60/60 + business_hours = round(duration_day_zero + complete_days_hours + duration_day_n, 2) + return max(0, business_hours) + +$$; + +{% endmacro %} diff --git a/macros/udfs/business_hours.yml b/macros/udfs/business_hours.yml new file mode 100644 index 0000000..9134826 --- /dev/null +++ b/macros/udfs/business_hours.yml @@ -0,0 +1,49 @@ +version: 2 + +macros: + - name: create_udf_business_hours + description: > + This macro creates a UDF within the target schema that can be used to + calculate business hours (excluding holidays) between two date fields + in a Snowflake table. The macro mearly creates the UDF within Snowflake, + and is called by the create_udfs macro. You must ensure that create_udfs + is called on-run-start in dbt_project.yml. + + To utilize the UDF within your dbt project, proper syntax is: + + select + *, + udf_business_hours(start_datetime, end_datetime, country, weekmask) + as business_hours_elapsed + from sample_table + + Notes: + - The date fields passed MUST be type == datetime (i.e. timestamp_ntz). + - The country field is optional as it is defaulted to 'US'. This can be a + column within your table if you have businesses in different geographic + locations, or can be set as a string for the entire dataset. If looking + to use a different country, must use the ISO 3166-1 alpha-2 country code + as defined here: + https://en.wikipedia.org/wiki/List_of_ISO_3166_country_codes + - The weekmask field is a string indicating which days of the week are to + be considered business days, and defaults to the standard workweek being + Monday through Friday via weekmask = '1111100'; the weekmask starting + with Monday and ending with Sunday, with a 1 indicating viable business + day and 0 indicating a non-business day. This field is optional. + - Regarding defaulted arguments,calls can be made with the following + syntax: + udf_business_hours(start_datetime, end_datetime, country, weekmask) + udf_business_hours(start_datetime, end_datetime, country) + udf_business_hours(start_datetime, end_datetime) + - This UDF only considers holidays between Jan 1 1990 and Dec 31 2030, as + is currently hardcoded. This can be changed in the macro source code. + - If increased granularity is desired for the region, you must modify the + macro to apply regional (state or province) holidays as described as + defined here: + holiday_list = holidays.country_holidays(country, subdiv='PR') + - The holidays python package does not include the end date in its + calculations of the duration. To account for this, you can add a day to + your end date to make the calculation inclusive. + + docs: + show: false