From eff7a69aff68b686098ee9612f1c58036eec1e04 Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Mon, 3 Aug 2020 19:37:44 -0400 Subject: [PATCH 1/8] Replace adapter_macro with adapter.dispatch --- dbt_project.yml | 2 +- integration_tests/Makefile | 34 ++++++++++++------ .../cross_db_utils/_get_utils_namespaces.sql | 4 +++ macros/cross_db_utils/concat.sql | 10 +++--- macros/cross_db_utils/current_timestamp.sql | 4 +-- macros/cross_db_utils/datatypes.sql | 12 +++---- macros/cross_db_utils/date_trunc.sql | 2 +- macros/cross_db_utils/dateadd.sql | 9 +---- macros/cross_db_utils/datediff.sql | 35 +------------------ macros/cross_db_utils/except.sql | 2 +- macros/cross_db_utils/hash.sql | 2 +- macros/cross_db_utils/identifier.sql | 2 +- macros/cross_db_utils/intersect.sql | 2 +- macros/cross_db_utils/last_day.sql | 2 +- macros/cross_db_utils/length.sql | 2 +- macros/cross_db_utils/literal.sql | 2 +- macros/cross_db_utils/position.sql | 2 +- macros/cross_db_utils/replace.sql | 2 +- macros/cross_db_utils/right.sql | 2 +- macros/cross_db_utils/safe_cast.sql | 2 +- macros/cross_db_utils/split_part.sql | 2 +- macros/cross_db_utils/width_bucket.sql | 2 +- macros/sql/get_tables_by_prefix_sql.sql | 2 +- 23 files changed, 58 insertions(+), 82 deletions(-) create mode 100644 macros/cross_db_utils/_get_utils_namespaces.sql diff --git a/dbt_project.yml b/dbt_project.yml index 03a57409..e9a847d3 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,7 +1,7 @@ name: 'dbt_utils' version: '0.1.0' -require-dbt-version: [">=0.17.0", "<0.18.0"] +require-dbt-version: [">=0.18.0", "<0.19.0"] config-version: 2 target-path: "target" diff --git a/integration_tests/Makefile b/integration_tests/Makefile index 89cc0fbc..325ed1d9 100644 --- a/integration_tests/Makefile +++ b/integration_tests/Makefile @@ -1,14 +1,26 @@ -TARGETS = postgres redshift snowflake bigquery -.PHONY : test test-all $(TARGETS) +test-postgres: + dbt seed --target postgres --full-refresh + dbt run --target postgres --full-refresh --vars 'update: false' + dbt run --target postgres --vars 'update: true' + dbt test --target postgres -test: export TARGET = $(target) -test: export MODELS = $(models) -test: export SEEDS = $(seeds) -test: - docker-compose -f ../docker-compose.yml up dbt +test-redshift: + dbt seed --target redshift --full-refresh + dbt run --target redshift --full-refresh --vars 'update: false' + dbt run --target redshift --vars 'update: true' + dbt test --target redshift -$(TARGETS): - $(MAKE) test target=$@ +test-snowflake: + dbt seed --target snowflake --full-refresh + dbt run --target snowflake --full-refresh --vars 'update: false' + dbt run --target snowflake --vars 'update: true' + dbt test --target snowflake -test-all: $(TARGETS) - echo "Tested all targets" +test-bigquery: + dbt seed --target bigquery --full-refresh + dbt run --target bigquery --full-refresh --vars 'update: false' + dbt run --target bigquery --vars 'update: true' + dbt test --target bigquery + +test-all: test-postgres test-redshift test-snowflake test-bigquery + echo "Completed successfully" diff --git a/macros/cross_db_utils/_get_utils_namespaces.sql b/macros/cross_db_utils/_get_utils_namespaces.sql new file mode 100644 index 00000000..4296f86e --- /dev/null +++ b/macros/cross_db_utils/_get_utils_namespaces.sql @@ -0,0 +1,4 @@ +{% macro _get_utils_namespaces() %} + {% set override_namespaces = var('utils_namespaces', []) %} + {% do return(override_namespaces + ['dbt_utils']) %} +{% endmacro %} diff --git a/macros/cross_db_utils/concat.sql b/macros/cross_db_utils/concat.sql index 88512611..07bc6483 100644 --- a/macros/cross_db_utils/concat.sql +++ b/macros/cross_db_utils/concat.sql @@ -1,23 +1,23 @@ + + {% macro concat(fields) -%} - {{ adapter_macro('dbt_utils.concat', fields) }} + {{ adapter.dispatch('concat', packages = dbt_utils._get_utils_namespaces())(fields) }} {%- endmacro %} - {% macro default__concat(fields) -%} concat({{ fields|join(', ') }}) {%- endmacro %} - {% macro alternative_concat(fields) %} {{ fields|join(' || ') }} {% endmacro %} {% macro redshift__concat(fields) %} - {{dbt_utils.alternative_concat(fields)}} + {{ alternative_concat(fields) }} {% endmacro %} {% macro snowflake__concat(fields) %} - {{dbt_utils.alternative_concat(fields)}} + {{ alternative_concat(fields) }} {% endmacro %} diff --git a/macros/cross_db_utils/current_timestamp.sql b/macros/cross_db_utils/current_timestamp.sql index d838dce9..090e246d 100644 --- a/macros/cross_db_utils/current_timestamp.sql +++ b/macros/cross_db_utils/current_timestamp.sql @@ -1,5 +1,5 @@ {% macro current_timestamp() -%} - {{ adapter_macro('dbt_utils.current_timestamp') }} + {{ adapter.dispatch('current_timestamp', packages = dbt_utils._get_utils_namespaces())() }} {%- endmacro %} {% macro default__current_timestamp() %} @@ -17,7 +17,7 @@ {% macro current_timestamp_in_utc() -%} - {{ adapter_macro('dbt_utils.current_timestamp_in_utc') }} + {{ adapter.dispatch('current_timestamp_in_utc', packages = dbt_utils._get_utils_namespaces())() }} {%- endmacro %} {% macro default__current_timestamp_in_utc() %} diff --git a/macros/cross_db_utils/datatypes.sql b/macros/cross_db_utils/datatypes.sql index 24632ddd..1f949281 100644 --- a/macros/cross_db_utils/datatypes.sql +++ b/macros/cross_db_utils/datatypes.sql @@ -1,7 +1,7 @@ {# string ------------------------------------------------- #} {%- macro type_string() -%} - {{ adapter_macro('dbt_utils.type_string') }} + {{ adapter.dispatch('type_string', packages = dbt_utils._get_utils_namespaces())() }} {%- endmacro -%} {% macro default__type_string() %} @@ -25,7 +25,7 @@ {# timestamp ------------------------------------------------- #} {%- macro type_timestamp() -%} - {{ adapter_macro('dbt_utils.type_timestamp') }} + {{ adapter.dispatch('type_timestamp', packages = dbt_utils._get_utils_namespaces())() }} {%- endmacro -%} {% macro default__type_timestamp() %} @@ -40,7 +40,7 @@ {# float ------------------------------------------------- #} {%- macro type_float() -%} - {{ adapter_macro('dbt_utils.type_float') }} + {{ adapter.dispatch('type_float', packages = dbt_utils._get_utils_namespaces())() }} {%- endmacro -%} {% macro default__type_float() %} @@ -54,7 +54,7 @@ {# numeric ------------------------------------------------ #} {%- macro type_numeric() -%} - {{ adapter_macro('dbt_utils.type_numeric') }} + {{ adapter.dispatch('type_numeric', packages = dbt_utils._get_utils_namespaces())() }} {%- endmacro -%} {% macro default__type_numeric() %} @@ -69,7 +69,7 @@ {# bigint ------------------------------------------------- #} {%- macro type_bigint() -%} - {{ adapter_macro('dbt_utils.type_bigint') }} + {{ adapter.dispatch('type_bigint', packages = dbt_utils._get_utils_namespaces())() }} {%- endmacro -%} {% macro default__type_bigint() %} @@ -83,7 +83,7 @@ {# int ------------------------------------------------- #} {%- macro type_int() -%} - {{ adapter_macro('dbt_utils.type_int') }} + {{ adapter.dispatch('type_int', packages = dbt_utils._get_utils_namespaces())() }} {%- endmacro -%} {% macro default__type_int() %} diff --git a/macros/cross_db_utils/date_trunc.sql b/macros/cross_db_utils/date_trunc.sql index d6073739..08653134 100644 --- a/macros/cross_db_utils/date_trunc.sql +++ b/macros/cross_db_utils/date_trunc.sql @@ -1,5 +1,5 @@ {% macro date_trunc(datepart, date) -%} - {{ adapter_macro('dbt_utils.date_trunc', datepart, date) }} + {{ adapter.dispatch('date_trunc', packages = dbt_utils._get_utils_namespaces()) (datepart, date) }} {%- endmacro %} {% macro default__date_trunc(datepart, date) %} diff --git a/macros/cross_db_utils/dateadd.sql b/macros/cross_db_utils/dateadd.sql index 92977df8..8df7fc77 100644 --- a/macros/cross_db_utils/dateadd.sql +++ b/macros/cross_db_utils/dateadd.sql @@ -1,5 +1,5 @@ {% macro dateadd(datepart, interval, from_date_or_timestamp) %} - {{ adapter_macro('dbt_utils.dateadd', datepart, interval, from_date_or_timestamp) }} + {{ adapter.dispatch('dateadd', packages = dbt_utils._get_utils_namespaces())(datepart, interval, from_date_or_timestamp) }} {% endmacro %} @@ -22,10 +22,3 @@ ) {% endmacro %} - - -{% macro postgres__dateadd(datepart, interval, from_date_or_timestamp) %} - - {{ from_date_or_timestamp }} + ((interval '1 {{ datepart }}') * ({{ interval }})) - -{% endmacro %} diff --git a/macros/cross_db_utils/datediff.sql b/macros/cross_db_utils/datediff.sql index c0c1e7d4..44c6052f 100644 --- a/macros/cross_db_utils/datediff.sql +++ b/macros/cross_db_utils/datediff.sql @@ -1,5 +1,5 @@ {% macro datediff(first_date, second_date, datepart) %} - {{ adapter_macro('dbt_utils.datediff', first_date, second_date, datepart) }} + {{ adapter.dispatch('datediff', packages = dbt_utils._get_utils_namespaces())(first_date, second_date, datepart) }} {% endmacro %} @@ -24,36 +24,3 @@ {% endmacro %} - -{% macro postgres__datediff(first_date, second_date, datepart) %} - - {% if datepart == 'year' %} - (date_part('year', ({{second_date}})::date) - date_part('year', ({{first_date}})::date)) - {% elif datepart == 'quarter' %} - ({{ dbt_utils.datediff(first_date, second_date, 'year') }} * 4 + date_part('quarter', ({{second_date}})::date) - date_part('quarter', ({{first_date}})::date)) - {% elif datepart == 'month' %} - ({{ dbt_utils.datediff(first_date, second_date, 'year') }} * 12 + date_part('month', ({{second_date}})::date) - date_part('month', ({{first_date}})::date)) - {% elif datepart == 'day' %} - (({{second_date}})::date - ({{first_date}})::date) - {% elif datepart == 'week' %} - ({{ dbt_utils.datediff(first_date, second_date, 'day') }} / 7 + case - when date_part('dow', ({{first_date}})::timestamp) <= date_part('dow', ({{second_date}})::timestamp) then - case when {{first_date}} <= {{second_date}} then 0 else -1 end - else - case when {{first_date}} <= {{second_date}} then 1 else 0 end - end) - {% elif datepart == 'hour' %} - ({{ dbt_utils.datediff(first_date, second_date, 'day') }} * 24 + date_part('hour', ({{second_date}})::timestamp) - date_part('hour', ({{first_date}})::timestamp)) - {% elif datepart == 'minute' %} - ({{ dbt_utils.datediff(first_date, second_date, 'hour') }} * 60 + date_part('minute', ({{second_date}})::timestamp) - date_part('minute', ({{first_date}})::timestamp)) - {% elif datepart == 'second' %} - ({{ dbt_utils.datediff(first_date, second_date, 'minute') }} * 60 + floor(date_part('second', ({{second_date}})::timestamp)) - floor(date_part('second', ({{first_date}})::timestamp))) - {% elif datepart == 'millisecond' %} - ({{ dbt_utils.datediff(first_date, second_date, 'minute') }} * 60000 + floor(date_part('millisecond', ({{second_date}})::timestamp)) - floor(date_part('millisecond', ({{first_date}})::timestamp))) - {% elif datepart == 'microsecond' %} - ({{ dbt_utils.datediff(first_date, second_date, 'minute') }} * 60000000 + floor(date_part('microsecond', ({{second_date}})::timestamp)) - floor(date_part('microsecond', ({{first_date}})::timestamp))) - {% else %} - {{ exceptions.raise_compiler_error("Unsupported datepart for macro datediff in postgres: {!r}".format(datepart)) }} - {% endif %} - -{% endmacro %} diff --git a/macros/cross_db_utils/except.sql b/macros/cross_db_utils/except.sql index 95a022f2..c9e8206d 100644 --- a/macros/cross_db_utils/except.sql +++ b/macros/cross_db_utils/except.sql @@ -1,5 +1,5 @@ {% macro except() %} - {{ adapter_macro('dbt_utils.except') }} + {{ adapter.dispatch('except', packages = dbt_utils._get_utils_namespaces())() }} {% endmacro %} diff --git a/macros/cross_db_utils/hash.sql b/macros/cross_db_utils/hash.sql index b7d2c235..8244b7f8 100644 --- a/macros/cross_db_utils/hash.sql +++ b/macros/cross_db_utils/hash.sql @@ -1,5 +1,5 @@ {% macro hash(field) -%} - {{ adapter_macro('dbt_utils.hash', field) }} + {{ adapter.dispatch('hash', packages = dbt_utils._get_utils_namespaces()) (field) }} {%- endmacro %} diff --git a/macros/cross_db_utils/identifier.sql b/macros/cross_db_utils/identifier.sql index ede09f66..1a379261 100644 --- a/macros/cross_db_utils/identifier.sql +++ b/macros/cross_db_utils/identifier.sql @@ -4,7 +4,7 @@ Use `adapter.quote` instead. The {}.{} model triggered this warning. \ '.format(model.package_name, model.name) -%} {%- do exceptions.warn(error_message) -%} - {{ adapter_macro('dbt_utils.identifier', value) }} + {{ adapter.dispatch('identifier', packages = dbt_utils._get_utils_namespaces()) (value) }} {% endmacro %} {% macro default__identifier(value) -%} diff --git a/macros/cross_db_utils/intersect.sql b/macros/cross_db_utils/intersect.sql index c2853bd0..252479a0 100644 --- a/macros/cross_db_utils/intersect.sql +++ b/macros/cross_db_utils/intersect.sql @@ -1,5 +1,5 @@ {% macro intersect() %} - {{ adapter_macro('dbt_utils.intersect') }} + {{ adapter.dispatch('intersect', packages = dbt_utils._get_utils_namespaces())() }} {% endmacro %} diff --git a/macros/cross_db_utils/last_day.sql b/macros/cross_db_utils/last_day.sql index 0efe6257..1a58b51b 100644 --- a/macros/cross_db_utils/last_day.sql +++ b/macros/cross_db_utils/last_day.sql @@ -4,7 +4,7 @@ testing is required to validate that it will work on other dateparts. */ {% macro last_day(date, datepart) %} - {{ adapter_macro('dbt_utils.last_day', date, datepart) }} + {{ adapter.dispatch('last_day', packages = dbt_utils._get_utils_namespaces()) (date, datepart) }} {% endmacro %} diff --git a/macros/cross_db_utils/length.sql b/macros/cross_db_utils/length.sql index a2bfc8f0..a2595d2e 100644 --- a/macros/cross_db_utils/length.sql +++ b/macros/cross_db_utils/length.sql @@ -1,5 +1,5 @@ {% macro length(expression) -%} - {{ adapter_macro('dbt_utils.length', expression) }} + {{ adapter.dispatch('length', packages = dbt_utils._get_utils_namespaces()) (expression) }} {% endmacro %} diff --git a/macros/cross_db_utils/literal.sql b/macros/cross_db_utils/literal.sql index 2d29b36f..b486fc2f 100644 --- a/macros/cross_db_utils/literal.sql +++ b/macros/cross_db_utils/literal.sql @@ -1,6 +1,6 @@ {%- macro string_literal(value) -%} - {{ adapter_macro('dbt_utils.string_literal', value) }} + {{ adapter.dispatch('string_literal', packages = dbt_utils._get_utils_namespaces()) (value) }} {%- endmacro -%} {% macro default__string_literal(value) -%} diff --git a/macros/cross_db_utils/position.sql b/macros/cross_db_utils/position.sql index df4a58bb..a5b70c3a 100644 --- a/macros/cross_db_utils/position.sql +++ b/macros/cross_db_utils/position.sql @@ -1,5 +1,5 @@ {% macro position(substring_text, string_text) -%} - {{ adapter_macro('dbt_utils.position', substring_text, string_text) }} + {{ adapter.dispatch('position', packages = dbt_utils._get_utils_namespaces()) (substring_text, string_text) }} {% endmacro %} diff --git a/macros/cross_db_utils/replace.sql b/macros/cross_db_utils/replace.sql index 00b2837d..f1dc363d 100644 --- a/macros/cross_db_utils/replace.sql +++ b/macros/cross_db_utils/replace.sql @@ -1,5 +1,5 @@ {% macro replace(field, old_chars, new_chars) -%} - {{ adapter_macro('dbt_utils.replace', field, old_chars, new_chars) }} + {{ adapter.dispatch('replace', packages = dbt_utils._get_utils_namespaces()) (field, old_chars, new_chars) }} {% endmacro %} diff --git a/macros/cross_db_utils/right.sql b/macros/cross_db_utils/right.sql index 5a2d5226..992b848b 100644 --- a/macros/cross_db_utils/right.sql +++ b/macros/cross_db_utils/right.sql @@ -1,5 +1,5 @@ {% macro right(string_text, length_expression) -%} - {{ adapter_macro('dbt_utils.right', string_text, length_expression) }} + {{ adapter.dispatch('right', packages = dbt_utils._get_utils_namespaces()) (string_text, length_expression) }} {% endmacro %} {% macro default__right(string_text, length_expression) %} diff --git a/macros/cross_db_utils/safe_cast.sql b/macros/cross_db_utils/safe_cast.sql index 592a0d39..1c3ef532 100644 --- a/macros/cross_db_utils/safe_cast.sql +++ b/macros/cross_db_utils/safe_cast.sql @@ -1,5 +1,5 @@ {% macro safe_cast(field, type) %} - {{ adapter_macro('dbt_utils.safe_cast', field, type) }} + {{ adapter.dispatch('safe_cast', packages = dbt_utils._get_utils_namespaces()) (field, type) }} {% endmacro %} diff --git a/macros/cross_db_utils/split_part.sql b/macros/cross_db_utils/split_part.sql index 6434cf91..0f1b2f22 100644 --- a/macros/cross_db_utils/split_part.sql +++ b/macros/cross_db_utils/split_part.sql @@ -1,5 +1,5 @@ {% macro split_part(string_text, delimiter_text, part_number) %} - {{ adapter_macro('dbt_utils.split_part', string_text, delimiter_text, part_number) }} + {{ adapter.dispatch('split_part', packages = dbt_utils._get_utils_namespaces()) (string_text, delimiter_text, part_number) }} {% endmacro %} diff --git a/macros/cross_db_utils/width_bucket.sql b/macros/cross_db_utils/width_bucket.sql index 39e0b0b6..8e714410 100644 --- a/macros/cross_db_utils/width_bucket.sql +++ b/macros/cross_db_utils/width_bucket.sql @@ -1,5 +1,5 @@ {% macro width_bucket(expr, min_value, max_value, num_buckets) %} - {{ adapter_macro('dbt_utils.width_bucket', expr, min_value, max_value, num_buckets) }} + {{ adapter.dispatch('width_bucket', packages = dbt_utils._get_utils_namespaces()) (expr, min_value, max_value, num_buckets) }} {% endmacro %} diff --git a/macros/sql/get_tables_by_prefix_sql.sql b/macros/sql/get_tables_by_prefix_sql.sql index 2f70bf95..e054012a 100644 --- a/macros/sql/get_tables_by_prefix_sql.sql +++ b/macros/sql/get_tables_by_prefix_sql.sql @@ -1,5 +1,5 @@ {% macro get_tables_by_prefix_sql(schema, prefix, exclude='', database=target.database) %} - {{ adapter_macro('dbt_utils.get_tables_by_prefix_sql', schema, prefix, exclude, database) }} + {{ adapter.dispatch('get_tables_by_prefix_sql', packages = dbt_utils._get_utils_namespaces()) (schema, prefix, exclude, database) }} {% endmacro %} {% macro default__get_tables_by_prefix_sql(schema, prefix, exclude='', database=target.database) %} From 0e8e70386821d0d50a0d85e5d4e9ceb370e72d30 Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Fri, 14 Aug 2020 19:54:58 -0400 Subject: [PATCH 2/8] Refactor to get tests passing --- .../models/datetime/test_date_spine.sql | 3 +- .../sql/test_get_relations_by_pattern.sql | 2 +- macros/cross_db_utils/concat.sql | 4 +-- macros/sql/get_relations_by_pattern.sql | 16 ++++++++-- macros/sql/get_tables_by_pattern.sql | 12 ------- macros/sql/get_tables_by_pattern_sql.sql | 28 +++++++++++++++++ macros/sql/get_tables_by_prefix_sql.sql | 31 +++++-------------- 7 files changed, 53 insertions(+), 43 deletions(-) delete mode 100644 macros/sql/get_tables_by_pattern.sql create mode 100644 macros/sql/get_tables_by_pattern_sql.sql diff --git a/integration_tests/models/datetime/test_date_spine.sql b/integration_tests/models/datetime/test_date_spine.sql index 4d0d4004..93cd07f1 100644 --- a/integration_tests/models/datetime/test_date_spine.sql +++ b/integration_tests/models/datetime/test_date_spine.sql @@ -9,8 +9,7 @@ with date_spine as ( {% if target.type == 'postgres' %} - {{ log("WARNING: Not testing - datediff macro is unsupported on Postgres", info=True) }} - select * from {{ ref('data_date_spine') }} + {{ dbt_utils.date_spine("day", "'2018-01-01'::date", "'2018-01-10'::date") }} {% elif target.type == 'bigquery' %} select cast(date_day as date) as date_day diff --git a/integration_tests/models/sql/test_get_relations_by_pattern.sql b/integration_tests/models/sql/test_get_relations_by_pattern.sql index 5a51dd05..b10d8eef 100644 --- a/integration_tests/models/sql/test_get_relations_by_pattern.sql +++ b/integration_tests/models/sql/test_get_relations_by_pattern.sql @@ -1,4 +1,4 @@ {{ config(materialized = 'table') }} -{% set relations = dbt_utils.get_relations_by_pattern(target.schema, 'data_events_') %} +{% set relations = dbt_utils.get_relations_by_pattern(target.schema, 'data_events_%') %} {{ dbt_utils.union_relations(relations) }} \ No newline at end of file diff --git a/macros/cross_db_utils/concat.sql b/macros/cross_db_utils/concat.sql index 07bc6483..58435016 100644 --- a/macros/cross_db_utils/concat.sql +++ b/macros/cross_db_utils/concat.sql @@ -14,10 +14,10 @@ {% macro redshift__concat(fields) %} - {{ alternative_concat(fields) }} + {{ dbt_utils.alternative_concat(fields) }} {% endmacro %} {% macro snowflake__concat(fields) %} - {{ alternative_concat(fields) }} + {{ dbt_utils.alternative_concat(fields) }} {% endmacro %} diff --git a/macros/sql/get_relations_by_pattern.sql b/macros/sql/get_relations_by_pattern.sql index 78a766ad..e09a6aba 100644 --- a/macros/sql/get_relations_by_pattern.sql +++ b/macros/sql/get_relations_by_pattern.sql @@ -2,7 +2,7 @@ {%- call statement('get_tables', fetch_result=True) %} - {{ dbt_utils.get_tables_by_pattern(schema_pattern, table_pattern, exclude, database) }} + {{ dbt_utils.get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude, database) }} {%- endcall -%} @@ -20,4 +20,16 @@ {{ return([]) }} {%- endif -%} -{% endmacro %} \ No newline at end of file +{% endmacro %} + +{% macro get_tables_by_pattern(schema_pattern, table_pattern, exclude='', database=target.database) %} + {%- set error_message = ' + Warning: the `get_tables_by_pattern` macro is no longer supported and will be deprecated in a future release of dbt-utils. \ + Use the `get_relations_by_prefix` macro instead. \ + The {}.{} model triggered this warning. \ + '.format(model.package_name, model.name) -%} + {%- do exceptions.warn(error_message) -%} + + {{ return(dbt_utils.get_relations_by_pattern(schema_pattern, table_pattern, exclude='', database=target.database)) }} + +{% endmacro %} diff --git a/macros/sql/get_tables_by_pattern.sql b/macros/sql/get_tables_by_pattern.sql deleted file mode 100644 index ad388ca4..00000000 --- a/macros/sql/get_tables_by_pattern.sql +++ /dev/null @@ -1,12 +0,0 @@ -{% macro get_tables_by_pattern(schema_pattern, table_pattern, exclude='', database=target.database) %} - - select distinct - table_schema as "table_schema", table_name as "table_name" - from {{database}}.information_schema.tables - where table_schema ilike '{{ schema_pattern }}' - and table_name ilike '{{ table_pattern }}' - and table_name not ilike '{{ exclude }}' - -{% endmacro %} - - diff --git a/macros/sql/get_tables_by_pattern_sql.sql b/macros/sql/get_tables_by_pattern_sql.sql new file mode 100644 index 00000000..df47aa23 --- /dev/null +++ b/macros/sql/get_tables_by_pattern_sql.sql @@ -0,0 +1,28 @@ +{% macro get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude='', database=target.database) %} + {{ adapter.dispatch('get_tables_by_pattern_sql', packages = dbt_utils._get_utils_namespaces()) + (schema_pattern, table_pattern, exclude='', database=target.database) }} +{% endmacro %} + +{% macro default__get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude='', database=target.database) %} + + select distinct + table_schema as "table_schema", table_name as "table_name" + from {{database}}.information_schema.tables + where table_schema ilike '{{ schema_pattern }}' + and table_name ilike '{{ table_pattern }}' + and table_name not ilike '{{ exclude }}' + +{% endmacro %} + + +{% macro bigquery__get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude='', database=target.database) %} + + select distinct + table_schema, table_name + + from {{adapter.quote(database)}}.{{schema}}.INFORMATION_SCHEMA.TABLES + where table_schema = '{{schema_pattern}}' + and lower(table_name) like lower ('{{table_pattern}}') + and lower(table_name) not like lower ('{{exclude}}') + +{% endmacro %} diff --git a/macros/sql/get_tables_by_prefix_sql.sql b/macros/sql/get_tables_by_prefix_sql.sql index e054012a..46e52221 100644 --- a/macros/sql/get_tables_by_prefix_sql.sql +++ b/macros/sql/get_tables_by_prefix_sql.sql @@ -1,27 +1,10 @@ {% macro get_tables_by_prefix_sql(schema, prefix, exclude='', database=target.database) %} - {{ adapter.dispatch('get_tables_by_prefix_sql', packages = dbt_utils._get_utils_namespaces()) (schema, prefix, exclude, database) }} -{% endmacro %} - -{% macro default__get_tables_by_prefix_sql(schema, prefix, exclude='', database=target.database) %} - - select distinct - table_schema as "table_schema", table_name as "table_name" - from {{database}}.information_schema.tables - where table_schema ilike '{{ schema }}' - and table_name ilike '{{ prefix }}%' - and table_name not ilike '{{ exclude }}' - -{% endmacro %} - - -{% macro bigquery__get_tables_by_prefix_sql(schema, prefix, exclude='', database=target.database) %} - select distinct - table_schema, table_name - - from {{adapter.quote(database)}}.{{schema}}.INFORMATION_SCHEMA.TABLES - where table_schema = '{{schema}}' - and lower(table_name) like lower ('{{prefix}}%') - and lower(table_name) not like lower ('{{exclude}}') - + {{ dbt_utils.get_tables_by_pattern_sql( + schema_pattern = schema, + table_pattern = prefix ~ '%', + exclude = exclude, + database = database + ) }} + {% endmacro %} From 4dd1bc3bbfca406336d82a046bd6bbf82e4e4992 Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Fri, 14 Aug 2020 19:55:57 -0400 Subject: [PATCH 3/8] Prefigure rc1 --- run_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_test.sh b/run_test.sh index 737e571a..6bede664 100755 --- a/run_test.sh +++ b/run_test.sh @@ -6,7 +6,7 @@ if [[ ! -f $VENV ]]; then . $VENV pip install --upgrade pip setuptools - pip install "dbt>=0.17.0,<0.18.0" + pip install "dbt==0.18.0rc1" fi . $VENV From 9e9407b9de7c48fa320098580a2aac7aa16551b9 Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Fri, 14 Aug 2020 20:03:00 -0400 Subject: [PATCH 4/8] Rename namespace var --- macros/cross_db_utils/_get_utils_namespaces.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/cross_db_utils/_get_utils_namespaces.sql b/macros/cross_db_utils/_get_utils_namespaces.sql index 4296f86e..e841381c 100644 --- a/macros/cross_db_utils/_get_utils_namespaces.sql +++ b/macros/cross_db_utils/_get_utils_namespaces.sql @@ -1,4 +1,4 @@ {% macro _get_utils_namespaces() %} - {% set override_namespaces = var('utils_namespaces', []) %} + {% set override_namespaces = var('dbt_utils_dispatch_list', []) %} {% do return(override_namespaces + ['dbt_utils']) %} {% endmacro %} From 25eba99bfc84233d511493220178dd10628611d9 Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Mon, 17 Aug 2020 10:01:20 -0400 Subject: [PATCH 5/8] Revert makefile changes --- integration_tests/Makefile | 34 +++++++++++----------------------- 1 file changed, 11 insertions(+), 23 deletions(-) diff --git a/integration_tests/Makefile b/integration_tests/Makefile index 325ed1d9..89cc0fbc 100644 --- a/integration_tests/Makefile +++ b/integration_tests/Makefile @@ -1,26 +1,14 @@ -test-postgres: - dbt seed --target postgres --full-refresh - dbt run --target postgres --full-refresh --vars 'update: false' - dbt run --target postgres --vars 'update: true' - dbt test --target postgres +TARGETS = postgres redshift snowflake bigquery +.PHONY : test test-all $(TARGETS) -test-redshift: - dbt seed --target redshift --full-refresh - dbt run --target redshift --full-refresh --vars 'update: false' - dbt run --target redshift --vars 'update: true' - dbt test --target redshift +test: export TARGET = $(target) +test: export MODELS = $(models) +test: export SEEDS = $(seeds) +test: + docker-compose -f ../docker-compose.yml up dbt -test-snowflake: - dbt seed --target snowflake --full-refresh - dbt run --target snowflake --full-refresh --vars 'update: false' - dbt run --target snowflake --vars 'update: true' - dbt test --target snowflake +$(TARGETS): + $(MAKE) test target=$@ -test-bigquery: - dbt seed --target bigquery --full-refresh - dbt run --target bigquery --full-refresh --vars 'update: false' - dbt run --target bigquery --vars 'update: true' - dbt test --target bigquery - -test-all: test-postgres test-redshift test-snowflake test-bigquery - echo "Completed successfully" +test-all: $(TARGETS) + echo "Tested all targets" From 9acf4b21d00588aa08a3aa581c7d014f9840dd9e Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Thu, 20 Aug 2020 12:08:13 -0400 Subject: [PATCH 6/8] Update _is_ephemeral while we're here --- macros/cross_db_utils/_is_ephemeral.sql | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/macros/cross_db_utils/_is_ephemeral.sql b/macros/cross_db_utils/_is_ephemeral.sql index e13be407..34c70db9 100644 --- a/macros/cross_db_utils/_is_ephemeral.sql +++ b/macros/cross_db_utils/_is_ephemeral.sql @@ -1,7 +1,8 @@ {% macro _is_ephemeral(obj, macro) %} {%- if obj.is_cte -%} - {% if obj.name.startswith('__dbt__CTE__') %} - {% set model_name = obj.name[12:] %} + {% set ephemeral_prefix = api.Relation.add_ephemeral_prefix('') %} + {% if obj.name.startswith(ephemeral_prefix) %} + {% set model_name = obj.name[(ephemeral_prefix|length):] %} {% else %} {% set model_name = obj.name %} {%- endif -%} From fecd4a40db7625a5cc29a7d2ab36863c29a9fdb2 Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Thu, 20 Aug 2020 12:13:11 -0400 Subject: [PATCH 7/8] Lost & found: pg dateadd, datediff --- macros/cross_db_utils/dateadd.sql | 6 ++++++ macros/cross_db_utils/datediff.sql | 32 ++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/macros/cross_db_utils/dateadd.sql b/macros/cross_db_utils/dateadd.sql index 8df7fc77..e4a7eb43 100644 --- a/macros/cross_db_utils/dateadd.sql +++ b/macros/cross_db_utils/dateadd.sql @@ -22,3 +22,9 @@ ) {% endmacro %} + +{% macro postgres__dateadd(datepart, interval, from_date_or_timestamp) %} + + {{ from_date_or_timestamp }} + ((interval '1 {{ datepart }}') * ({{ interval }})) + +{% endmacro %} diff --git a/macros/cross_db_utils/datediff.sql b/macros/cross_db_utils/datediff.sql index 44c6052f..a4cb4d6a 100644 --- a/macros/cross_db_utils/datediff.sql +++ b/macros/cross_db_utils/datediff.sql @@ -24,3 +24,35 @@ {% endmacro %} +{% macro postgres__datediff(first_date, second_date, datepart) %} + + {% if datepart == 'year' %} + (date_part('year', ({{second_date}})::date) - date_part('year', ({{first_date}})::date)) + {% elif datepart == 'quarter' %} + ({{ dbt_utils.datediff(first_date, second_date, 'year') }} * 4 + date_part('quarter', ({{second_date}})::date) - date_part('quarter', ({{first_date}})::date)) + {% elif datepart == 'month' %} + ({{ dbt_utils.datediff(first_date, second_date, 'year') }} * 12 + date_part('month', ({{second_date}})::date) - date_part('month', ({{first_date}})::date)) + {% elif datepart == 'day' %} + (({{second_date}})::date - ({{first_date}})::date) + {% elif datepart == 'week' %} + ({{ dbt_utils.datediff(first_date, second_date, 'day') }} / 7 + case + when date_part('dow', ({{first_date}})::timestamp) <= date_part('dow', ({{second_date}})::timestamp) then + case when {{first_date}} <= {{second_date}} then 0 else -1 end + else + case when {{first_date}} <= {{second_date}} then 1 else 0 end + end) + {% elif datepart == 'hour' %} + ({{ dbt_utils.datediff(first_date, second_date, 'day') }} * 24 + date_part('hour', ({{second_date}})::timestamp) - date_part('hour', ({{first_date}})::timestamp)) + {% elif datepart == 'minute' %} + ({{ dbt_utils.datediff(first_date, second_date, 'hour') }} * 60 + date_part('minute', ({{second_date}})::timestamp) - date_part('minute', ({{first_date}})::timestamp)) + {% elif datepart == 'second' %} + ({{ dbt_utils.datediff(first_date, second_date, 'minute') }} * 60 + floor(date_part('second', ({{second_date}})::timestamp)) - floor(date_part('second', ({{first_date}})::timestamp))) + {% elif datepart == 'millisecond' %} + ({{ dbt_utils.datediff(first_date, second_date, 'minute') }} * 60000 + floor(date_part('millisecond', ({{second_date}})::timestamp)) - floor(date_part('millisecond', ({{first_date}})::timestamp))) + {% elif datepart == 'microsecond' %} + ({{ dbt_utils.datediff(first_date, second_date, 'minute') }} * 60000000 + floor(date_part('microsecond', ({{second_date}})::timestamp)) - floor(date_part('microsecond', ({{first_date}})::timestamp))) + {% else %} + {{ exceptions.raise_compiler_error("Unsupported datepart for macro datediff in postgres: {!r}".format(datepart)) }} + {% endif %} + +{% endmacro %} From 9760ba1e8ec596ffa9accb9a56da759b55626f36 Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Fri, 21 Aug 2020 11:32:17 -0400 Subject: [PATCH 8/8] Add to readme, changelog --- CHANGELOG.md | 6 ++++++ README.md | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8648e52b..e5d73fb8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ ## Features +* Switch usage of `adapter_macro` to `adapter.dispatch`, and define `dbt_utils_dispatch_list`, +enabling users of community-supported database plugins to add or override macro implementations +specific to their database (#267) +* Use `add_ephemeral_prefix` instead of hard-coding a string literal, to support +database adapters that use different prefixes (#267) + ## Quality of life # dbt-utils v0.5.1 diff --git a/README.md b/README.md index 30607d4e..0300d1cb 100644 --- a/README.md +++ b/README.md @@ -777,6 +777,42 @@ We welcome contributions to this repo! To contribute a new feature or a fix, ple ---- +### Dispatch macros + +**Note:** This is primarily relevant to users and maintainers of community-supported +database plugins. If you use Postgres, Redshift, Snowflake, or Bigquery, this likely +does not apply to you. + +dbt v0.18.0 introduces `adapter.dispatch()`, a reliable way to define different implementations of the same macro +across different databases. + +All dispatched macros in `dbt_utils` have an override setting: a `var` named +`dbt_utils_dispatch_list` that accepts a list of package names. If you set this +variable in your project, when dbt searches for implementations of a dispatched +`dbt_utils` macro, it will search through your listed packages _before_ using +the implementations defined in `dbt_utils`. + +Set the variable: +```yml +vars: + dbt_utils_dispatch_list: + - first_package_to_search # likely the name of your root project + - second_package_to_search # likely an "add-on" package, such as spark_utils + # dbt_utils is always the last place searched +``` + +When running on Spark, if dbt needs to dispatch `dbt_utils.datediff`, it will search for the following in order: +``` +first_package_to_search.spark__datediff +first_package_to_search.default__datediff +second_package_to_search.spark__datediff +second_package_to_search.default__datediff +dbt_utils.spark__datediff +dbt_utils.default__datediff +``` + +---- + ### Getting started with dbt - [What is dbt]?