diff --git a/README.md b/README.md index 1453237..6d5fabd 100644 --- a/README.md +++ b/README.md @@ -28,8 +28,13 @@ adapter and providing jinja context with further adapter/python methods. UI page. - Customized dbt Docs: Replace the default dbt documentation page with your own custom index.html file. - Run end to end ETL pipeline with dbt - using [dlt](https://dlthub.com/) [integration](https://github.com/memiiso/opendbt/issues/40) -- Register dbt callbacks within a dbt project to trigger custom actions or alerts based on selected dbt events. + using [dlt extract and load](https://dlthub.com/) [integration](https://github.com/memiiso/opendbt/issues/40) +- Register [dbt callbacks](https://docs.getdbt.com/reference/programmatic-invocations#registering-callbacks) within a + dbt project to trigger custom actions or alerts based on selected dbt events. +- Use multi project dbt-mesh setup, + using [cross-project references](https://docs.getdbt.com/docs/collaborate/govern/project-dependencies#how-to-write-cross-project-ref). + ex: `{{ ref('jaffle_finance', 'monthly_revenue') }}` + - This feature was only available in "dbt Cloud Enterprise" so far. For detailed examples, see: [examples](docs/EXAMPLES.md). diff --git a/tests/base_dbt_test.py b/tests/base_dbt_test.py index 69c7984..534a04e 100644 --- a/tests/base_dbt_test.py +++ b/tests/base_dbt_test.py @@ -3,9 +3,18 @@ from dbt.version import get_installed_version as get_dbt_version +from opendbt import OpenDbtCli + class BaseDbtTest(TestCase): RESOURCES_DIR = Path(__file__).parent.joinpath("resources") DBTCORE_DIR = RESOURCES_DIR.joinpath("dbtcore") DBTFINANCE_DIR = RESOURCES_DIR.joinpath("dbtfinance") DBT_VERSION = get_dbt_version() + + @classmethod + def setUpClass(cls): + dpf = OpenDbtCli(project_dir=BaseDbtTest.DBTFINANCE_DIR, profiles_dir=BaseDbtTest.DBTFINANCE_DIR) + dpc = OpenDbtCli(project_dir=BaseDbtTest.DBTCORE_DIR, profiles_dir=BaseDbtTest.DBTCORE_DIR) + dpf.invoke(args=["clean"]) + dpc.invoke(args=["clean"]) diff --git a/tests/resources/airflow/dags/dbt_mesh_workflow.py b/tests/resources/airflow/dags/dbt_mesh_workflow.py new file mode 100644 index 0000000..f26a7ae --- /dev/null +++ b/tests/resources/airflow/dags/dbt_mesh_workflow.py @@ -0,0 +1,28 @@ +from pathlib import Path + +from airflow import DAG +from airflow.utils.dates import days_ago + +from opendbt.airflow import OpenDbtAirflowProject + +default_args = { + 'owner': 'airflow', + 'depends_on_past': False, + 'email_on_failure': False, + 'email_on_retry': False, + 'retries': 1 +} + +with DAG( + dag_id='dbt_mesh_workflow', + default_args=default_args, + description='DAG To run multiple dbt projects', + schedule_interval=None, + start_date=days_ago(3), + catchup=False, + max_active_runs=1 +) as dag: + DBT_PROJ_DIR = Path("/opt/dbtfinance") + + p = OpenDbtAirflowProject(project_dir=DBT_PROJ_DIR, profiles_dir=DBT_PROJ_DIR, target='dev') + p.load_dbt_tasks(dag=dag, include_singular_tests=True, include_dbt_seeds=True) diff --git a/tests/resources/airflow/docker-compose.yaml b/tests/resources/airflow/docker-compose.yaml index 36cea37..e5c796b 100644 --- a/tests/resources/airflow/docker-compose.yaml +++ b/tests/resources/airflow/docker-compose.yaml @@ -16,6 +16,7 @@ services: - ./dags:/opt/airflow/dags:rw - ./plugins:/opt/airflow/plugins:rw - ./../dbtcore:/opt/dbtcore:rw + - ./../dbtfinance:/opt/dbtfinance:rw - ./../../../opendbt/macros:/opt/dbtcore/macros:rw environment: - AIRFLOW__WEBSERVER__INSTANCE_NAME=LOCAL diff --git a/tests/resources/dbtcore/dbt_project.yml b/tests/resources/dbtcore/dbt_project.yml index c9426ff..f5dc7a3 100644 --- a/tests/resources/dbtcore/dbt_project.yml +++ b/tests/resources/dbtcore/dbt_project.yml @@ -4,16 +4,15 @@ version: '1.0.0' profile: 'dbtcore' macro-paths: [ "macros", "../../../opendbt/macros/" ] -snapshot-paths: [ "snapshots" ] docs-paths: [ "../../../opendbt/docs/" ] -clean-targets: # directories to be removed by `dbt clean` +clean-targets: - "target" - "dbt_packages" + - "logs" models: dbtcore: - # Config indicated by + and applies to all files under models/example/ example: +materialized: view diff --git a/tests/resources/dbtfinance/dbt_project.yml b/tests/resources/dbtfinance/dbt_project.yml index a72dc87..e9426f7 100644 --- a/tests/resources/dbtfinance/dbt_project.yml +++ b/tests/resources/dbtfinance/dbt_project.yml @@ -6,4 +6,5 @@ profile: 'dbtfinance' # directories to be removed by `dbt clean` clean-targets: - "target" - - "dbt_packages" \ No newline at end of file + - "dbt_packages" + - "logs" \ No newline at end of file diff --git a/tests/test_opendbt_cli.py b/tests/test_opendbt_cli.py index 4e4365b..a031487 100644 --- a/tests/test_opendbt_cli.py +++ b/tests/test_opendbt_cli.py @@ -41,5 +41,5 @@ def test_cli_run_models(self): def test_cli_run_cross_project_ref_models(self): dpf = OpenDbtCli(project_dir=self.DBTFINANCE_DIR) dpc = OpenDbtCli(project_dir=self.DBTCORE_DIR) - dpc.invoke(args=['run', '--select', 'my_core_table1', "--profiles-dir", dpc.project_dir.as_posix()]) dpf.invoke(args=['run', '--select', 'my_cross_project_ref_model', "--profiles-dir", dpf.project_dir.as_posix()]) + dpc.invoke(args=['run', '--select', 'my_core_table1', "--profiles-dir", dpc.project_dir.as_posix()])