Feature/generate_model_import_ctes

README.md

@@ -6,6 +6,7 @@ Macros that generate dbt code, and log it to the command line.
 * [generate_source](#generate_source-source)
 * [generate_base_model](#generate_base_model-source)
 * [generate_model_yaml](#generate_model_yaml-source)
+* [generate_model_import_ctes](#generate_model_import_ctes-source)
 
 # Installation instructions
 New to dbt packages? Read more about them [here](https://docs.getdbt.com/docs/building-a-dbt-project/package-management/).
@@ -164,3 +165,27 @@ models:
 ```
 
 4. Paste the output in to a schema.yml file, and refactor as required.
+
+## generate_model_import_ctes ([source](macros/generate_model_import_ctes.sql))
+This macro generates the SQL for a given model with all references pulled up into import CTEs, which you can then paste back into the model.
+
+### Arguments:
+* `model_name` (required): The model table you wish to generate SQL with import CTEs for.
+* `leading_commas` (optional, default = false): Whether you want your commas to be leading (vs trailing).
+
+### Usage:
+1. Copy the macro into a statement tab in the dbt Cloud IDE, or into an analysis file, and compile your code.
+
+```
+{{ generate_model_import_ctes(
+    model_name = 'my_dbt_model'
+) }}
+```
+
+Alternatively, call the macro as an [operation](https://docs.getdbt.com/docs/using-operations):
+
+```
+$ dbt run-operation generate_model_import_ctes --args '{"model_name": "my_dbt_model"}'
+```
+
+2. Replace the contents of the model's current SQL file with the compiled or logged code

integration_tests/models/model_without_any_ctes.sql

@@ -0,0 +1,3 @@
+select *, 2 as col2
+from {{ ref('model_without_import_ctes') }}
+where id = 1

integration_tests/models/model_without_import_ctes.sql

@@ -0,0 +1,38 @@
+/*
+    This is my model!
+*/
+
+{{ config(
+    materialized='table',
+) }}
+
+with my_first_cte as (
+    select
+        a.col_a,
+        b.col_b
+    from {{ ref('data__a_relation') }} as a
+    left join      {{ ref('data__b_relation') }} as b
+    on a.col_a = b.col_a
+    left join {{ ref('data__a_relation') }} as aa
+    on a.col_a = aa.col_a
+),
+my_second_cte as (
+    select
+        1 as id
+    from codegen_integration_tests__data_source_schema.codegen_integration_tests__data_source_table
+    union all
+    select
+        2 as id
+    from {{ source('codegen_integration_tests__data_source_schema', 'codegen_integration_tests__data_source_table_case_sensitive') }}  
+)
+-- my_third_cte as (
+--     select
+--         a.col_a,
+--         b.col_b
+--     from `raw_relation_1` as a
+--     left join "raw_relation_2" as b
+--     on a.col_a = b.col_b
+--     left join [raw_relation_3] as aa
+--     on a.col_a = aa.col_b
+-- )
+select * from my_second_cte

integration_tests/tests/test_generate_model_import_ctes.sql

@@ -0,0 +1,87 @@
+{% set actual_model_with_import_ctes = codegen.generate_model_import_ctes(
+    model_name = 'model_without_import_ctes'
+  )
+%}
+
+{% set expected_model_with_import_ctes %}
+/*
+    This is my model!
+*/
+
+{% raw %}{{ config(
+    materialized='table',
+) }}{% endraw %}
+
+with codegen_integration_tests__data_source_table as (
+
+    select * from codegen_integration_tests__data_source_schema.codegen_integration_tests__data_source_table
+    -- CAUTION: It's best practice to use the ref or source function instead of a direct reference
+
+),
+codegen_integration_tests__data_source_table_case_sensitive as (
+
+    select * from {% raw %}{{ source('codegen_integration_tests__data_source_schema', 'codegen_integration_tests__data_source_table_case_sensitive') }}{% endraw %} 
+    -- CAUTION: It's best practice to create staging layer for raw sources
+
+),
+data__a_relation as (
+
+    select * from {% raw %}{{ ref('data__a_relation') }}{% endraw %}
+
+),
+data__b_relation as (
+
+    select * from {% raw %}{{ ref('data__b_relation') }}{% endraw %}
+
+),
+raw_relation_1 as (
+
+    select * from `raw_relation_1`
+    -- CAUTION: It's best practice to use the ref or source function instead of a direct reference
+
+),
+raw_relation_2 as (
+
+    select * from "raw_relation_2"
+    -- CAUTION: It's best practice to use the ref or source function instead of a direct reference
+
+),
+raw_relation_3 as (
+
+    select * from [raw_relation_3]
+    -- CAUTION: It's best practice to use the ref or source function instead of a direct reference
+
+),
+my_first_cte as (
+    select
+        a.col_a,
+        b.col_b
+    from data__a_relation as a
+    left join      data__b_relation as b
+    on a.col_a = b.col_a
+    left join data__a_relation as aa
+    on a.col_a = aa.col_a
+),
+my_second_cte as (
+    select
+        1 as id
+    from codegen_integration_tests__data_source_table
+    union all
+    select
+        2 as id
+    from codegen_integration_tests__data_source_table_case_sensitive  
+)
+-- my_third_cte as (
+--     select
+--         a.col_a,
+--         b.col_b
+--     from raw_relation_1 as a
+--     left join raw_relation_2 as b
+--     on a.col_a = b.col_b
+--     left join raw_relation_3 as aa
+--     on a.col_a = aa.col_b
+-- )
+select * from my_second_cte
+{% endset %}
+
+{{ assert_equal (actual_model_with_import_ctes | trim, expected_model_with_import_ctes | trim) }}

integration_tests/tests/test_generate_model_import_ctes_leading.sql

@@ -0,0 +1,88 @@
+{% set actual_model_with_import_ctes = codegen.generate_model_import_ctes(
+    model_name = 'model_without_import_ctes',
+    leading_commas = true
+  )
+%}
+
+{% set expected_model_with_import_ctes %}
+/*
+    This is my model!
+*/
+
+{% raw %}{{ config(
+    materialized='table',
+) }}{% endraw %}
+
+with codegen_integration_tests__data_source_table as (
+
+    select * from codegen_integration_tests__data_source_schema.codegen_integration_tests__data_source_table
+    -- CAUTION: It's best practice to use the ref or source function instead of a direct reference
+
+)
+,codegen_integration_tests__data_source_table_case_sensitive as (
+
+    select * from {% raw %}{{ source('codegen_integration_tests__data_source_schema', 'codegen_integration_tests__data_source_table_case_sensitive') }}{% endraw %} 
+    -- CAUTION: It's best practice to create staging layer for raw sources
+
+)
+,data__a_relation as (
+
+    select * from {% raw %}{{ ref('data__a_relation') }}{% endraw %}
+
+)
+,data__b_relation as (
+
+    select * from {% raw %}{{ ref('data__b_relation') }}{% endraw %}
+
+)
+,raw_relation_1 as (
+
+    select * from `raw_relation_1`
+    -- CAUTION: It's best practice to use the ref or source function instead of a direct reference
+
+)
+,raw_relation_2 as (
+
+    select * from "raw_relation_2"
+    -- CAUTION: It's best practice to use the ref or source function instead of a direct reference
+
+)
+,raw_relation_3 as (
+
+    select * from [raw_relation_3]
+    -- CAUTION: It's best practice to use the ref or source function instead of a direct reference
+
+)
+,my_first_cte as (
+    select
+        a.col_a,
+        b.col_b
+    from data__a_relation as a
+    left join      data__b_relation as b
+    on a.col_a = b.col_a
+    left join data__a_relation as aa
+    on a.col_a = aa.col_a
+),
+my_second_cte as (
+    select
+        1 as id
+    from codegen_integration_tests__data_source_table
+    union all
+    select
+        2 as id
+    from codegen_integration_tests__data_source_table_case_sensitive  
+)
+-- my_third_cte as (
+--     select
+--         a.col_a,
+--         b.col_b
+--     from raw_relation_1 as a
+--     left join raw_relation_2 as b
+--     on a.col_a = b.col_b
+--     left join raw_relation_3 as aa
+--     on a.col_a = aa.col_b
+-- )
+select * from my_second_cte
+{% endset %}
+
+{{ assert_equal (actual_model_with_import_ctes | trim, expected_model_with_import_ctes | trim) }}

integration_tests/tests/test_generate_model_import_ctes_no_ctes.sql

@@ -0,0 +1,17 @@
+{% set actual_model_with_import_ctes = codegen.generate_model_import_ctes(
+    model_name = 'model_without_any_ctes'
+  )
+%}
+
+{% set expected_model_with_import_ctes %}
+with model_without_import_ctes as (
+
+    select * from {% raw %}{{ ref('model_without_import_ctes') }}{% endraw %}
+
+)
+select *, 2 as col2
+from model_without_import_ctes
+where id = 1
+{% endset %}
+
+{{ assert_equal (actual_model_with_import_ctes | trim, expected_model_with_import_ctes | trim) }}

integration_tests/tests/test_generate_source_some_tables.sql

@@ -14,7 +14,7 @@
 version: 2
 
 sources:
-  - name: {{ raw_schema | trim }}
+  - name: {{ raw_schema | trim | lower }}
     description: ""
     tables:
       - name: data__a_relation

integration_tests/tests/test_generate_source_table_descriptions.sql

@@ -8,7 +8,7 @@
 version: 2
 
 sources:
-  - name: {{ raw_schema | trim }}
+  - name: {{ raw_schema | trim | lower }}
     description: ""
     tables:
       - name: data__a_relation

integration_tests/tests/test_generate_source_table_name.sql

@@ -9,7 +9,7 @@ version: 2
 
 sources:
   - name: raw
-    schema: {{ raw_schema | trim }}
+    schema: {{ raw_schema | trim | lower }}
     tables:
       - name: data__a_relation
       - name: data__b_relation