Add API for substitution and refs

Signed-off-by: Bernát Gábor <bgabor8@bloomberg.net>

src/tox/config/loader/toml/__init__.py

@@ -3,13 +3,15 @@
 from pathlib import Path
 from typing import TYPE_CHECKING, Dict, Iterator, List, Mapping, Set, TypeVar, cast
 
-from tox.config.loader.api import Loader, Override
+from tox.config.loader.api import ConfigLoadArgs, Loader, Override
 from tox.config.types import Command, EnvList
 
 from ._api import TomlTypes
+from ._replace import unroll_refs_and_apply_substitutions
 from ._validate import validate
 
 if TYPE_CHECKING:
+    from tox.config.loader.convert import Factory
     from tox.config.loader.section import Section
     from tox.config.main import Config
 
@@ -37,6 +39,18 @@ def __repr__(self) -> str:
     def load_raw(self, key: str, conf: Config | None, env_name: str | None) -> TomlTypes:  # noqa: ARG002
         return self.content[key]
 
+    def build(  # noqa: PLR0913
+        self,
+        key: str,  # noqa: ARG002
+        of_type: type[_T],
+        factory: Factory[_T],
+        conf: Config | None,
+        raw: TomlTypes,
+        args: ConfigLoadArgs,
+    ) -> _T:
+        raw = unroll_refs_and_apply_substitutions(conf=conf, loader=self, value=raw, args=args)
+        return self.to(raw, of_type, factory)
+
     def found_keys(self) -> set[str]:
         return set(self.content.keys()) - self._unused_exclude
 

src/tox/config/loader/toml/_replace.py

@@ -0,0 +1,54 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Final
+
+if TYPE_CHECKING:
+    from tox.config.loader.api import ConfigLoadArgs
+    from tox.config.loader.toml import TomlLoader
+    from tox.config.main import Config
+
+    from ._api import TomlTypes
+
+MAX_REPLACE_DEPTH: Final[int] = 100
+
+
+class MatchRecursionError(ValueError):
+    """Could not stabilize on replacement value."""
+
+
+def unroll_refs_and_apply_substitutions(
+    conf: Config | None,
+    loader: TomlLoader,
+    value: TomlTypes,
+    args: ConfigLoadArgs,
+    depth: int = 0,
+) -> TomlTypes:
+    """Replace all active tokens within value according to the config."""
+    if depth > MAX_REPLACE_DEPTH:
+        msg = f"Could not expand {value} after recursing {depth} frames"
+        raise MatchRecursionError(msg)
+
+    if isinstance(value, str):
+        pass  # apply string substitution here
+    elif isinstance(value, (int, float, bool)):
+        pass  # no reference or substitution possible
+    elif isinstance(value, list):
+        # need to inspect every entry of the list to check for reference.
+        res_list: list[TomlTypes] = []
+        for val in value:  # apply replacement for every entry
+            got = unroll_refs_and_apply_substitutions(conf, loader, val, args, depth + 1)
+            res_list.append(got)
+        value = res_list
+    elif isinstance(value, dict):
+        # need to inspect every entry of the list to check for reference.
+        res_dict: dict[str, TomlTypes] = {}
+        for key, val in value.items():  # apply replacement for every entry
+            got = unroll_refs_and_apply_substitutions(conf, loader, val, args, depth + 1)
+            res_dict[key] = got
+        value = res_dict
+    return value
+
+
+__all__ = [
+    "unroll_refs_and_apply_substitutions",
+]

tests/config/source/test_toml_tox.py

@@ -63,5 +63,5 @@ def test_config_in_toml_extra(tox_project: ToxProjectCreator) -> None:
 
 def test_config_in_toml_replace(tox_project: ToxProjectCreator) -> None:
     project = tox_project({"tox.toml": '[env_run_base]\ndescription = "Magic in {env_name}"'})
-    outcome = project.run("c", "-k", "commands")
+    outcome = project.run("c", "-k", "description")
     outcome.assert_success()

tox.toml

@@ -0,0 +1,131 @@
+requires = ["tox>=4.19"]
+env_list = ["fix", "3.13", "3.12", "3.11", "3.10", "3.9", "3.8", "cov", "type", "docs", "pkg_meta"]
+skip_missing_interpreters = true
+
+[env_run_base]
+description = "run the tests with pytest under {env_name}"
+package = "wheel"
+wheel_build_env = ".pkg"
+extras = ["testing"]
+pass_env = ["PYTEST_*", "SSL_CERT_FILE"]
+set_env.COVERAGE_FILE = { type = "env", name = "COVERAGE_FILE", default = "{work_dir}{/}.coverage.{env_name}" }
+set_env.COVERAGE_FILECOVERAGE_PROCESS_START = "{tox_root}{/}pyproject.toml"
+commands = [
+  [
+    "pytest",
+    { type = "posargs", default = [
+      "--junitxml",
+      "{work_dir}{/}junit.{env_name}.xml",
+      "--cov",
+      "{env_site_packages_dir}{/}tox",
+      "--cov",
+      "{tox_root}{/}tests",
+      "--cov-config={tox_root}{/}pyproject.toml",
+      "-no-cov-on-fail",
+      "--cov-report",
+      "term-missing:skip-covered",
+      "--cov-context=test",
+      "--cov-report",
+      "html:{env_tmp_dir}{/}htmlcov",
+      "--cov-report",
+      "xml:{work_dir}{/}coverage.{env_name}.xml",
+      "-n",
+      { type = "env", name = "PYTEST_XDIST_AUTO_NUM_WORKERS", default = "auto" },
+      "tests",
+      "--durations",
+      "15",
+      "--run-integration",
+    ] },
+  ],
+  [
+    "diff-cover",
+    "--compare-branch",
+    { type = "env", name = "DIFF_AGAINST", default = "origin/main" },
+    "{work_dir}{/}coverage.{env_name}.xml",
+  ],
+]
+
+[env.fix]
+description = "format the code base to adhere to our styles, and complain about what we cannot do automatically"
+skip_install = true
+deps = ["pre-commit-uv>=4.1.3"]
+pass_env = [{ type = "ref", of = ["env_run_base", "pass_env"] }, "PROGRAMDATA"]
+commands = [["pre-commit", "run", "--all-files", "--show-diff-on-failure", { type = "posargs" }]]
+
+[env.type]
+description = "run type check on code base"
+deps = ["mypy==1.11.2", "types-cachetools>=5.5.0.20240820", "types-chardet>=5.0.4.6"]
+commands = [["mypy", "src/tox"], ["mypy", "tests"]]
+
+[env.docs]
+description = "build documentation"
+extras = ["docs"]
+commands = [
+  { type = "posargs", default = [
+    "sphinx-build",
+    "-d",
+    "{env_tmp_dir}{/}docs_tree",
+    "docs",
+    "{work_dir}{/}docs_out",
+    "--color",
+    "-b",
+    "linkcheck",
+  ] },
+  [
+    "sphinx-build",
+    "-d",
+    "{env_tmp_dir}{/}docs_tree",
+    "docs",
+    "{work_dir}{/}docs_out",
+    "--color",
+    "-b",
+    "html",
+    "-W",
+  ],
+  [
+    "python",
+    "-c",
+    'print(r"documentation available under file://{work_dir}{/}docs_out{/}index.html")',
+  ],
+]
+
+
+[env.pkg_meta]
+description = "check that the long description is valid"
+skip_install = true
+deps = ["check-wheel-contents>=0.6", "twine>=5.1.1", "uv>=0.4.17"]
+commands = [
+  [
+    "uv",
+    "build",
+    "--sdist",
+    "--wheel",
+    "--out-dir",
+    "{env_tmp_dir}",
+    ".",
+  ],
+  [
+    "twine",
+    "check",
+    "{env_tmp_dir}{/}*",
+  ],
+  [
+    "check-wheel-contents",
+    "--no-config",
+    "{env_tmp_dir}",
+  ],
+]
+
+[env.release]
+description = "do a release, required posargs of the version number"
+skip_install = true
+deps = ["gitpython>=3.1.43", "packaging>=24.1", "towncrier>=24.8"]
+commands = [["python", "{tox_root}/tasks/release.py", "--version", "{posargs}"]]
+
+[env.dev]
+description = "dev environment with all deps at {envdir}"
+package = "editable"
+deps = { type = "ref", of = ["env", "release", "deps"] }
+extras = ["docs", "testing"]
+commands = [["python", "-m", "pip", "list", "--format=columns"], ["python", "-c", 'print(r"{env_python}")']]
+uv_seed = true