From 2d094438fca7badebb27a33b146e2e275f119833 Mon Sep 17 00:00:00 2001 From: ismail simsek <6005685+ismailsimsek@users.noreply.github.com> Date: Mon, 16 Dec 2024 22:40:42 +0100 Subject: [PATCH] Enable multi project references and settings --- opendbt/dbt/__init__.py | 2 ++ opendbt/dbt/v18/config/__init__.py | 0 opendbt/dbt/v18/config/runtime.py | 53 ++++++++++++++++++++++++++++++ 3 files changed, 55 insertions(+) create mode 100644 opendbt/dbt/v18/config/__init__.py create mode 100644 opendbt/dbt/v18/config/runtime.py diff --git a/opendbt/dbt/__init__.py b/opendbt/dbt/__init__.py index f30766f..4682364 100644 --- a/opendbt/dbt/__init__.py +++ b/opendbt/dbt/__init__.py @@ -24,6 +24,8 @@ def patch_dbt(): dbt.adapters.factory.FACTORY = OpenDbtAdapterContainer() from opendbt.dbt.v18.task.run import ModelRunner dbt.task.run.ModelRunner = ModelRunner + from opendbt.dbt.v18.config.runtime import OpenDbtRuntimeConfig + dbt.config.RuntimeConfig = OpenDbtRuntimeConfig else: raise Exception( f"Unsupported dbt version {dbt_version}, please make sure dbt version is supported/integrated by opendbt") diff --git a/opendbt/dbt/v18/config/__init__.py b/opendbt/dbt/v18/config/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/opendbt/dbt/v18/config/runtime.py b/opendbt/dbt/v18/config/runtime.py new file mode 100644 index 0000000..be71f0c --- /dev/null +++ b/opendbt/dbt/v18/config/runtime.py @@ -0,0 +1,53 @@ +from dataclasses import dataclass +from pathlib import Path +from typing import Mapping + +from dbt.config import RuntimeConfig +from dbt.config.project import load_yml_dict +from dbt.constants import DEPENDENCIES_FILE_NAME +from dbt.exceptions import ( + DbtProjectError, NonUniquePackageNameError, +) +from typing_extensions import override + + +# pylint: disable=too-many-ancestors +@dataclass +class OpenDbtRuntimeConfig(RuntimeConfig): + def load_dependence_projects(self): + dependencies_yml_dict = load_yml_dict(f"{self.project_root}/{DEPENDENCIES_FILE_NAME}") + + if "projects" not in dependencies_yml_dict: + return + + projects = dependencies_yml_dict["projects"] + project_root_parent = Path(self.project_root).parent + for project in projects: + path = project_root_parent.joinpath(project['name']) + try: + project = self.new_project(str(path.as_posix())) + except DbtProjectError as e: + raise DbtProjectError( + f"Failed to read depending project: {e} \n project path:{path.as_posix()}", + result_type="invalid_project", + path=path, + ) from e + + yield project.project_name, project + + @override + def load_dependencies(self, base_only=False) -> Mapping[str, "RuntimeConfig"]: + # if self.dependencies is None: + + if self.dependencies is None: + # this sets self.dependencies variable! + self.dependencies = super().load_dependencies() + + # additionally load `projects` defined in `dependencies.yml` + for project_name, project in self.load_dependence_projects(): + print(f"WARN: loading:{project_name} , project:{project}") + if project_name in self.dependencies: + raise NonUniquePackageNameError(project_name) + self.dependencies[project_name] = project + + return self.dependencies